diff options
600 files changed, 13240 insertions, 8841 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 28c1814998..55cebc008f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,37 +1,21 @@ -<!--- Remove sections that do not apply --> +<!-- This issue tracker is only for technical issues related to Bitcoin Core. -This issue tracker is only for technical issues related to bitcoin-core. +General bitcoin questions and/or support requests are best directed to the Bitcoin StackExchange at https://bitcoin.stackexchange.com. -General bitcoin questions and/or support requests and are best directed to the [Bitcoin StackExchange](https://bitcoin.stackexchange.com). +For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/. -For reporting security issues, please read instructions at [https://bitcoincore.org/en/contact/](https://bitcoincore.org/en/contact/). +If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running memtest and observe CPU temperature with a load-test tool such as linpack before creating an issue! --> -### Describe the issue +<!-- Describe the issue --> +<!--- What behavior did you expect? --> -### Can you reliably reproduce the issue? -#### If so, please list the steps to reproduce below: -1. -2. -3. +<!--- What was the actual behavior (provide screenshots if the issue is GUI-related)? --> -### Expected behaviour -Tell us what should happen +<!--- How reliably can you reproduce the issue, what are the steps to do so? --> -### Actual behaviour -Tell us what happens instead +<!-- What version of Bitcoin Core are you using, where did you get it (website, self-compiled, etc)? --> -### Screenshots. -If the issue is related to the GUI, screenshots can be added to this issue via drag & drop. +<!-- What type of machine are you observing the error on (OS/CPU and disk type)? --> -### What version of bitcoin-core are you using? -List the version number/commit ID, and if it is an official binary, self compiled or a distribution package such as PPA. - -### Machine specs: -- OS: -- CPU: -- RAM: -- Disk size: -- Disk Type (HD/SDD): - -### Any extra information that might be useful in the debugging process. -This is normally the contents of a `debug.log` or `config.log` file. Raw text or a link to a pastebin type site are preferred. +<!-- Any extra information that might be useful in the debugging process. --> +<!--- This is normally the contents of a `debug.log` or `config.log` file. Raw text or a link to a pastebin type site are preferred. --> diff --git a/.gitignore b/.gitignore index 60c26dae8b..ff297fbeca 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,7 @@ src/qt/test/moc*.cpp Makefile bitcoin-qt Bitcoin-Qt.app +background.tiff* # Unit-tests Makefile.test diff --git a/.travis.yml b/.travis.yml index 0de7ca6f75..ab002acc53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: required dist: trusty os: linux -language: generic +language: minimal cache: directories: - depends/built @@ -18,11 +18,10 @@ env: - CCACHE_COMPRESS=1 - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - SDK_URL=https://bitcoincore.org/depends-sources/sdks - - PYTHON_DEBUG=1 - WINEDEBUG=fixme-all matrix: # ARM - - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf python3-pip" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" # Win32 - HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" # Qt4 & system libs @@ -40,16 +39,20 @@ env: before_install: - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") - - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/pyenv/d' | tr "\n" ":" | sed "s|::|:|g") install: - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi + - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then travis_retry pip3 install flake8 --user; fi before_script: - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then contrib/devtools/commit-script-check.sh $TRAVIS_COMMIT_RANGE; fi + - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/crypto/ctaes; fi + - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/secp256k1; fi + - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/univalue; fi + - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/git-subtree-check.sh src/leveldb; fi - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-rpc-mappings.py .; fi - - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/lint-all.sh; fi + - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then contrib/devtools/lint-all.sh; fi - unset CC; unset CXX - mkdir -p depends/SDKs depends/sdk-sources - if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi @@ -76,7 +79,7 @@ script: - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib - if [ "$RUN_TESTS" = "true" ]; then travis_wait 30 make $MAKEJOBS check VERBOSE=1; fi - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then extended="--extended --exclude pruning,dbcrash"; fi - - if [ "$RUN_TESTS" = "true" ]; then test/functional/test_runner.py --coverage --quiet ${extended}; fi + - if [ "$RUN_TESTS" = "true" ]; then test/functional/test_runner.py --combinedlogslen=4000 --coverage --quiet ${extended}; fi after_script: - echo $TRAVIS_COMMIT_RANGE - echo $TRAVIS_COMMIT_LOG diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aed6d79542..8381bd2448 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,9 +24,9 @@ facilitates social contribution, easy testing and peer review. To contribute a patch, the workflow is as follows: - - Fork repository - - Create topic branch - - Commit patches + 1. Fork repository + 1. Create topic branch + 1. Commit patches The project coding conventions in the [developer notes](doc/developer-notes.md) must be adhered to. @@ -42,8 +42,8 @@ in init.cpp") in which case a single title line is sufficient. Commit messages s helpful to people reading your code in the future, so explain the reasoning for your decisions. Further explanation [here](http://chris.beams.io/posts/git-commit/). -If a particular commit references another issue, please add the reference, for -example `refs #1234`, or `fixes #4321`. Using the `fixes` or `closes` keywords +If a particular commit references another issue, please add the reference. For +example: `refs #1234` or `fixes #4321`. Using the `fixes` or `closes` keywords will cause the corresponding issue to be closed when the pull request is merged. Please refer to the [Git manual](https://git-scm.com/doc) for more information @@ -85,7 +85,7 @@ Note that translations should not be submitted as pull requests, please see [Translation Process](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md) for more information on helping with translations. -If a pull request is specifically not to be considered for merging (yet) please +If a pull request is not to be considered for merging (yet), please prefix the title with [WIP] or use [Tasks Lists](https://help.github.com/articles/basic-writing-and-formatting-syntax/#task-lists) in the body of the pull request to indicate tasks are pending. @@ -186,7 +186,7 @@ In general, all pull requests must: the project (for example refactoring for modularisation); - Be well peer reviewed; - Have unit tests and functional tests where appropriate; - - Follow code style guidelines; + - Follow code style guidelines ([C++](doc/developer-notes.md), [functional tests](test/functional/README.md)); - Not break the existing test suite; - Where bugs are fixed, where possible, there should be unit tests demonstrating the bug and also proving the fix. This helps prevent regression. @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2009-2017 The Bitcoin Core developers -Copyright (c) 2009-2017 Bitcoin Developers +Copyright (c) 2009-2018 The Bitcoin Core developers +Copyright (c) 2009-2018 Bitcoin Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile.am b/Makefile.am index 3b62a10603..b24daf9905 100644 --- a/Makefile.am +++ b/Makefile.am @@ -9,7 +9,6 @@ SUBDIRS += doc/man endif .PHONY: deploy FORCE -GZIP_ENV="-9n" export PYTHONPATH if BUILD_BITCOIN_LIBS @@ -43,7 +42,11 @@ DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \ $(top_srcdir)/contrib/bitcoin-tx.bash-completion \ $(top_srcdir)/contrib/bitcoind.bash-completion \ $(top_srcdir)/contrib/init \ + $(top_srcdir)/contrib/install_db4.sh \ $(top_srcdir)/contrib/rpm +DIST_SHARE = \ + $(top_srcdir)/share/genbuild.sh \ + $(top_srcdir)/share/rpcauth BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ $(top_srcdir)/contrib/devtools/security-check.py @@ -213,7 +216,7 @@ endif dist_noinst_SCRIPTS = autogen.sh -EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) +EXTRA_DIST = $(DIST_SHARE) test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) EXTRA_DIST += \ test/util/bitcoin-util-test.py \ @@ -276,4 +279,4 @@ DISTCHECK_CONFIGURE_FLAGS = --enable-man clean-local: rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP) - rm -rf test/functional/__pycache__ + rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache diff --git a/configure.ac b/configure.ac index 81c84a8af4..256aed40b2 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ define(_CLIENT_VERSION_MINOR, 15) define(_CLIENT_VERSION_REVISION, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_IS_RELEASE, false) -define(_COPYRIGHT_YEAR, 2017) +define(_COPYRIGHT_YEAR, 2018) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Bitcoin Core]]) AC_INIT([Bitcoin Core],[_CLIENT_VERSION_MAJOR._CLIENT_VERSION_MINOR._CLIENT_VERSION_REVISION],[https://github.com/bitcoin/bitcoin/issues],[bitcoin],[https://bitcoincore.org/]) @@ -162,9 +162,9 @@ AC_ARG_ENABLE([ccache], AC_ARG_ENABLE([lcov], [AS_HELP_STRING([--enable-lcov], [enable lcov testing (default is no)])], - [use_lcov=yes], + [use_lcov=$enableval], [use_lcov=no]) - + AC_ARG_ENABLE([lcov-branch-coverage], [AS_HELP_STRING([--enable-lcov-branch-coverage], [enable lcov testing branch coverage (default is no)])], @@ -241,6 +241,7 @@ if test "x$enable_werror" = "xyes"; then AC_MSG_ERROR("enable-werror set but -Werror is not usable") fi AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=thread-safety-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety-analysis"],,[[$CXXFLAG_WERROR]]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -249,6 +250,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wformat],[CXXFLAGS="$CXXFLAGS -Wformat"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[CXXFLAGS="$CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wformat-security],[CXXFLAGS="$CXXFLAGS -Wformat-security"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wthread-safety-analysis],[CXXFLAGS="$CXXFLAGS -Wthread-safety-analysis"],,[[$CXXFLAG_WERROR]]) ## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all ## unknown options if any other warning is produced. Test the -Wfoo case, and @@ -657,6 +659,28 @@ AC_LINK_IFELSE([AC_LANG_SOURCE([ ] ) +TEMP_LDFLAGS="$LDFLAGS" +LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" +AC_MSG_CHECKING([for thread_local support]) +AC_LINK_IFELSE([AC_LANG_SOURCE([ + #include <thread> + static thread_local int foo = 0; + static void run_thread() { foo++;} + int main(){ + for(int i = 0; i < 10; i++) { std::thread(run_thread).detach();} + return foo; + } + ])], + [ + AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) + AC_MSG_RESULT(yes) + ], + [ + AC_MSG_RESULT(no) + ] +) +LDFLAGS="$TEMP_LDFLAGS" + # Check for different ways of gathering OS randomness AC_MSG_CHECKING(for Linux getrandom syscall) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> @@ -737,6 +761,9 @@ define(MINIMUM_REQUIRED_BOOST, 1.47.0) dnl Check for boost libs AX_BOOST_BASE([MINIMUM_REQUIRED_BOOST]) +if test x$want_boost = xno; then + AC_MSG_ERROR([[only libbitcoinconsensus can be built without boost]]) +fi AX_BOOST_SYSTEM AX_BOOST_FILESYSTEM AX_BOOST_PROGRAM_OPTIONS @@ -1286,7 +1313,7 @@ case ${OS} in ;; esac -echo +echo echo "Options used to compile and link:" echo " with wallet = $enable_wallet" echo " with gui / qt = $bitcoin_enable_qt" @@ -1301,7 +1328,7 @@ echo " with upnp = $use_upnp" echo " use asm = $use_asm" echo " debug enabled = $enable_debug" echo " werror = $enable_werror" -echo +echo echo " target os = $TARGET_OS" echo " build os = $BUILD_OS" echo @@ -1312,4 +1339,4 @@ echo " CXX = $CXX" echo " CXXFLAGS = $CXXFLAGS" echo " LDFLAGS = $LDFLAGS" echo " ARFLAGS = $ARFLAGS" -echo +echo diff --git a/contrib/debian/bitcoin-qt.desktop b/contrib/debian/bitcoin-qt.desktop index 593d7584ab..204cdf99d0 100644 --- a/contrib/debian/bitcoin-qt.desktop +++ b/contrib/debian/bitcoin-qt.desktop @@ -1,5 +1,5 @@ [Desktop Entry] -Encoding=UTF-8 +Version=1.0 Name=Bitcoin Core Comment=Connect to the Bitcoin P2P Network Comment[de]=Verbinde mit dem Bitcoin peer-to-peer Netzwerk @@ -11,3 +11,4 @@ Type=Application Icon=bitcoin128 MimeType=x-scheme-handler/bitcoin; Categories=Office;Finance; +StartupWMClass=Bitcoin-qt diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 72d64ce62d..c6484157a5 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -5,7 +5,7 @@ Upstream-Contact: Satoshi Nakamoto <satoshin@gmx.com> Source: https://github.com/bitcoin/bitcoin Files: * -Copyright: 2009-2017, Bitcoin Core Developers +Copyright: 2009-2018, Bitcoin Core Developers License: Expat Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, as well as the numerous contributors to the project. @@ -15,6 +15,14 @@ Copyright: 2010-2011, Jonas Smedegaard <dr@jones.dk> 2011, Matt Corallo <matt@bluematt.me> License: GPL-2+ +Files: src/secp256k1/build-aux/m4/ax_jni_include_dir.m4 +Copyright: 2008 Don Anderson <dda@sleepycat.com> +License: GNU-All-permissive-License + +Files: src/secp256k1/build-aux/m4/ax_prog_cc_for_build.m4 +Copyright: 2008 Paolo Bonzini <bonzini@gnu.org> +License: GNU-All-permissive-License + Files: src/qt/res/icons/add.png src/qt/res/icons/address-book.png src/qt/res/icons/chevron.png @@ -106,6 +114,12 @@ License: Expat TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +License: GNU-All-permissive-License + Copying and distribution of this file, with or without modification, are + permitted in any medium without royalty provided the copyright notice + and this notice are preserved. This file is offered as-is, without any + warranty. + License: GPL-2+ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the diff --git a/contrib/debian/examples/bitcoin.conf b/contrib/debian/examples/bitcoin.conf index 14a59fdf6b..4dd73162a2 100644 --- a/contrib/debian/examples/bitcoin.conf +++ b/contrib/debian/examples/bitcoin.conf @@ -77,9 +77,9 @@ #rpcpassword=YourSuperGreatPasswordNumber_DO_NOT_USE_THIS_OR_YOU_WILL_GET_ROBBED_385593 # # The second method `rpcauth` can be added to server startup argument. It is set at initialization time -# using the output from the script in share/rpcuser/rpcuser.py after providing a username: +# using the output from the script in share/rpcauth/rpcauth.py after providing a username: # -# ./share/rpcuser/rpcuser.py alice +# ./share/rpcauth/rpcauth.py alice # String to be appended to bitcoin.conf: # rpcauth=alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae # Your password: diff --git a/contrib/devtools/check-doc.py b/contrib/devtools/check-doc.py index 10c3bab03b..f164ea9322 100755 --- a/contrib/devtools/check-doc.py +++ b/contrib/devtools/check-doc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 6d801d3ac7..c817e794b9 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -24,8 +24,10 @@ EXCLUDE = [ 'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h', 'src/secp256k1/src/java/org_bitcoin_Secp256k1Context.c', 'src/secp256k1/src/java/org_bitcoin_Secp256k1Context.h', - # auto generated: + # univalue: + 'src/univalue/test/object.cpp', 'src/univalue/lib/univalue_escapes.h', + # auto generated: 'src/qt/bitcoinstrings.cpp', 'src/chainparamsseeds.h', # other external copyrights: diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index 967717e1e0..925d6a6252 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} SRCDIR=${SRCDIR:-$TOPDIR/src} diff --git a/contrib/devtools/git-subtree-check.sh b/contrib/devtools/git-subtree-check.sh index 2384d66cad..184951715e 100755 --- a/contrib/devtools/git-subtree-check.sh +++ b/contrib/devtools/git-subtree-check.sh @@ -18,7 +18,7 @@ find_latest_squash() sub= git log --grep="^git-subtree-dir: $dir/*\$" \ --pretty=format:'START %H%n%s%n%n%b%nEND%n' "$COMMIT" | - while read a b junk; do + while read a b _; do case "$a" in START) sq="$b" ;; git-subtree-mainline:) main="$b" ;; @@ -41,21 +41,17 @@ find_latest_squash() done } +# find latest subtree update latest_squash="$(find_latest_squash "$DIR")" if [ -z "$latest_squash" ]; then echo "ERROR: $DIR is not a subtree" >&2 exit 2 fi - set $latest_squash old=$1 rev=$2 -if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then - echo "ERROR: subtree commit $rev unavailable. Fetch/update the subtree repository" >&2 - exit 2 -fi -tree_subtree=$(git show -s --format="%T" $rev) -echo "$DIR in $COMMIT was last updated to upstream commit $rev (tree $tree_subtree)" + +# get the tree in the current commit tree_actual=$(git ls-tree -d "$COMMIT" "$DIR" | head -n 1) if [ -z "$tree_actual" ]; then echo "FAIL: subtree directory $DIR not found in $COMMIT" >&2 @@ -69,9 +65,30 @@ if [ "d$tree_actual_type" != "dtree" ]; then echo "FAIL: subtree directory $DIR is not a tree in $COMMIT" >&2 exit 1 fi + +# get the tree at the time of the last subtree update +tree_commit=$(git show -s --format="%T" $old) +echo "$DIR in $COMMIT was last updated in commit $old (tree $tree_commit)" + +# ... and compare the actual tree with it +if [ "$tree_actual_tree" != "$tree_commit" ]; then + git diff $tree_commit $tree_actual_tree >&2 + echo "FAIL: subtree directory was touched without subtree merge" >&2 + exit 1 +fi + +# get the tree in the subtree commit referred to +if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then + echo "subtree commit $rev unavailable: cannot compare" >&2 + exit +fi +tree_subtree=$(git show -s --format="%T" $rev) +echo "$DIR in $COMMIT was last updated to upstream commit $rev (tree $tree_subtree)" + +# ... and compare the actual tree with it if [ "$tree_actual_tree" != "$tree_subtree" ]; then - git diff-tree $tree_actual_tree $tree_subtree >&2 - echo "FAIL: subtree directory tree doesn't match subtree commit tree" >&2 + echo "FAIL: subtree update commit differs from upstream tree!" >&2 exit 1 fi + echo "GOOD" diff --git a/contrib/devtools/lint-python.sh b/contrib/devtools/lint-python.sh new file mode 100755 index 0000000000..9303fcc8ef --- /dev/null +++ b/contrib/devtools/lint-python.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for specified flake8 warnings in python files. + +# F401: module imported but unused +flake8 --ignore=B,C,E,F,I,N,W --select=F401 . diff --git a/contrib/devtools/optimize-pngs.py b/contrib/devtools/optimize-pngs.py index 9286ab731f..5cb3bb6f75 100755 --- a/contrib/devtools/optimize-pngs.py +++ b/contrib/devtools/optimize-pngs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 6eb5667453..1613f704df 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 18f9835faa..22f5ee20f7 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python2 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/gitian-build.sh b/contrib/gitian-build.sh index 8fdec21b0e..631fba9089 100755 --- a/contrib/gitian-build.sh +++ b/contrib/gitian-build.sh @@ -6,7 +6,6 @@ sign=false verify=false build=false -setupenv=false # Systems to build linux=true @@ -106,7 +105,7 @@ while :; do fi shift else - echo 'Error: "--os" requires an argument containing an l (for linux), w (for windows), or x (for Mac OSX)\n' + echo 'Error: "--os" requires an argument containing an l (for linux), w (for windows), or x (for Mac OSX)' exit 1 fi ;; @@ -189,7 +188,7 @@ then fi # Get signer -if [[ -n"$1" ]] +if [[ -n "$1" ]] then SIGNER=$1 shift diff --git a/contrib/gitian-keys/sjors-key.pgp b/contrib/gitian-keys/sjors-key.pgp new file mode 100644 index 0000000000..2b5acc82aa --- /dev/null +++ b/contrib/gitian-keys/sjors-key.pgp @@ -0,0 +1,76 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFWSwMoBEADG31O8+ex+xpgzVKQgF4iVRE5uBPT0+GM6FnwqIIhXVKiBLQh8 +YDhhgk6joh+vsLrFzKZ9kXwoiHN8y/AiNCQ0xjAUdpznD5xvHAaGIAlT/sodRNT+ +869WgT9G1uiVp0P4ucEeilmhCn9o51LqkS3roXkj0ec52b1pslUl2WKdu1ZD+Bj4 +3/oVZm7mmjkDwl0RHJQmqlK0bunq0jlVlgH5sdQfmLbCZaq3LhVPf73zt5qHH+J6 +ZbU7A4cqm2eN5SyH+Nno+cq3+vXmvVI+x/jPe/dPDCXaGWf5fWI/Lbk/mMP7JAl1 +6X44CN+hZHUnNuzeZt2/ROWZ0s0JJcjQkSe9noUQedjBAHX82s886vsFzOHvDtul +EuV/XAjUlkhMbhZkZaIq9ucqHmUBI4+OcFEIbbKc9TrKtJe+CYuWTNlomVk/iFr8 +zSm/S64NiqKi/BeQGgcsDZIaJDYfDP83esOOaaxFswHnJNtHnU1PwntrJtXft0dK +ydtlQZ6r96SYxLDTeGfC2SNk0zbnKAGvjj04vzQeN+JSRZ75tNKmgdbJdNL8wvPh +879TpCwMhNDvSRG+YqCe6whaJV76a+Doxg48HCJYaj6bnRn41/QGJEyL31I8l/7S +YsLLmAEbqwG7erYi7WZS3cRrGJI8RwohGMZf7yraqoaOgMKmtE/Sq0tLtwARAQAB +tCNTam9ycyBQcm92b29zdCA8c2pvcnNAc3Byb3Zvb3N0Lm5sPokCQAQTAQoAKgIb +AwUJB4YfgAULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAUCWWXcoAIZAQAKCRBX/5vb +zDAQCeYJD/47XDMfEMg4g4spo7k92XsNkvjlAhWvvxd+kxow/V8c64WQXody32FZ +HRSmK8dVjf9mIJMKkX4lpKpim7cQxsdTcorcdu+yk4TK+Wah61vsMhbSSllfHs1U ++q8jYMGnXTD+CY0aeTMrTfJcR2yN98jmNSWIL1qWmJ51RSTL6BQKb6eYtR7pWRkW +uMR6oFC09Db4fiKa4zhH81+/t0g+6pMY391gSluaS+OfNqGORCo+/IdG5IDzh5Vp +f19qXjd5oMsZQf6/P4b4XUktgl8RVRcNzdYGoXpcd8LpeHtEOh5I93ODmCwqd67b +YDlhDNN7iGhPndPEF6P4CNO/rXLPCZyMhRyt1dflu0KPCr+0AgR31cdhH/p7eCyj +FTE9gUgUHOG9OHdRoVXrwHYXwAiDBr2pp2giLpBsAwa4d2hXNDJ6wfMMCSOXKQlS +lHq06y/v/049DammkqW0XnEsU4qvsdteZ0jQu7Ob3LyGoytBIj8fn1OioT21W7wc +ns3/Tt4cQsn2ICBYB4PzqwkvGUp7fDwwHYw7rq6kvCEVDUDWMtVgQ8kjsh2OoU75 +eeteM1Q1fV06Wfn2Qct9bn0NKRGrA8mm3lrCWYCeGqJeBvC6kna1QgV53vYRLJod +w3Ql4+M9tUIi9uiGLvVaGZWO9wU1EwL+EAO+6D85h6QiJN7H8gcwUokCPQQTAQoA +JwUCVZLAygIbAwUJB4YfgAULCQgHAwUVCgkICwUWAgMBAAIeAQIXgAAKCRBX/5vb +zDAQCauuD/9IDWhf/fTseA1Rt5i4gwK+8dCQjTlRS2cZtGc2aMX8w5XruDWnna1P +Mj/aVUncDrprRx9rxgEqIDyPheuJ6r7v6D8GjrpAjcG/BPNFtPaxQccbZbAYdzoj +Rrs+ttVIqS+wO7qLmQkKA4oGRMmgYh3VX8EBZNcvxaGCcJx0PfoqS8cPXTnCRHcg +Wx6kaFyuWtrTX+kCpDraB1KGtxedR4rzuOtUOLoqFOOfsQuOxPlKNNr9Zjc8x2o4 +5TtwbuoEog8FIEttY6NOywpsSsvYvNB4gq1fxO49H0pQopmJlOMatMH6IRT7BJJZ +cOoHOh4X/zItOJZtuCOT4u+Y2XOuyLcW83X5ymIR3ZCxedsLzjyiCWm61/znJVON +Ws8I+gShbvauahBCB9rOHqwM0QioJMc36hUPB21KghQS8RJpGwmtk1WhFFMtAsSJ +w+wRfy2d6u+lSGdlA+2hEyKVm/DNQMDCQVFx3lQ6YBwAwkSiLMylrPKvs56fUjRr +74qoPyDxuRMC+q+TThHsy5O9r31G+Dc3+H5k4iTk354Jshjltx/k2O732e9Vxyar +/U5P7UZqHHuJKXDihUFrcJZq+gk8sGEWzGG/wocce7ezrTnHqR8YA04BTA4PXQqZ +4N42f422YYGIH/3Nm6drQkbigekLw6wx+NrxtTsYg4eCtSsaUd/RjLQhU2pvcnMg +UHJvdm9vc3QgPHNqb3JzQGZyZWVkb20ubmw+iQI9BBMBCgAnBQJZZdyfAhsDBQkH +hh+ABQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEFf/m9vMMBAJEsIQAK4ihgRB +05QqETpWNeV/XSGBHQINuwwEDz/k8dAJ5Uo6OoSpDULa16fs/EgAV46wTSxfWuci +n2Fc1AWLeLDWOax/NlycL00VDHEwT2PCjcc5uMuwR4RUTciKyByT1u7BFToZ6PyL +mbU6u6whcQejl6Ci2kw0Mu4n4bKTS7OL4/w/EbdfMSpRi8wWmTPMB/aMjtS2Mxi/ +N+yQhJ9pReHADeCBoAjq0cUy+QbzvBwDCK4XWRzF7kiFuA7UW2r7/dX6l31mPfi/ +GLA5+ftPxJ6EH8cxToF70OWiSfhOTleaqZaHUOG0V7wV2lr/bwAYzpVlxeZSCIta +lAA9ZLzUD2hiHYcei6kc/YjIhmlml7O0FK1eBk7+bt5wr0nvWt4Lbha4y5LxBX8C +d7InvB3xUYHz+S5Ul4vp0Rzx97MBL4oX2ltBEDpc1CcOgzv4dcWMG9bbh9/SaI/G +RehAzwkbpVUl9AEUNKO0dNlZUdu8CkehHdPdz5sJyS/9zE0A7yIECDFP9Nrht0nK +MahBijm4K+jOiLOZ2xyfOX1pVWLqIXGQHKjfcD3oI3qvGrQYtxB5Dffb9ACFMpZO +z3jM8h2UAa2/KqA4MZiZG9N6uWHKkIAMMuXWs1s439WePvbQ+5aw/qPUAMyqA3XZ +dkfn8QWaJPR4nRM+McYBYuS4fKK9HRJWQgcQuQINBFWSwMoBEACzmkabZ8oHWJUE +beU7rJF/TMbwV1IFtFxJ/QlY8rE4VnHekPMvkLi/gjx3WY5nmMe+d4JYoK/uPNdt +y5u0QYgH2MB/jebk4gYXCAHIPpU38h9UgHRb6qV8OaqHhmoXvKwyz+1QPzyJpmgg +oCUN+OAroNjl7zhunE7w7EEddFQftfPoGKEUnTjv84QOCuAb46JsYyiNAc3h6okq +74hY7PKCv8IRGclMPjemhBT2LEenn1t4yi7a8W/hjIe44PmQiqQEXR17keqcP/ls +EH9xSST1v/70ieiPqb6zbHGWzjQxqpFUJxRU6OluBCy5pHVd8wfFGYrrbTpoxaUC +jyA2SLr1oZZ9gaGprt6X7FC5gpE5LV9essq3O5wwvoPbyMe1F5uFaxIPhlt55oEu +rwVWecFJ8tSjniF/WSkTcILrOmiQZ4mylXfOP9Wk38seZReCs799KEfKFlXHk89a +Sj3ZvaJQxwVCnvsAsbVKmmHZ5wPt+G2KfhOkkv2A1I/UyeTT7aXvt2vxDqGuG0su +Eo6QknM/2Sr5Uv7BwBeSIQ6llH5ZnqKz34+HjriP8YPWzvsC959GXsxS01dCSvUM +92j5PvTZzf5dt1CWHMeufAY5XIH+nftkRniuScRhJ7xK3tJ7wngg7UvdeZwJWqmK +lJ7GI38V8HIMnd2x28yiGpj1ue6T+QARAQABiQIlBBgBCgAPBQJVksDKAhsMBQkH +hh+AAAoJEFf/m9vMMBAJjeIP/1UBCi6gSXzpGJBLD2u4PcZJjXBJAImZdf1aCqfS +YZBCaA65UrM3uaVa7h8MGAJc9kDjpqHurjDmG3YWf33KvHWYmReQvX43pZmfF12s +X7FZgcCfgZJKKj+ri6oHQonZzUMrecEcAJLLaQoD3Du3iZpETiyRLL7sJ1lZSaCJ +gYKnN4WV5GypvdFvb8vSUBST2h0D6AewGKMNh8ruRlkIxI+YSlywgYIH+O0qNKqW +wBlZc/5f+JZ3hu+cjx/+Zn+w+saIb6SgySg0UzN35b2WM2YzrfQep4ah3NIxuC7e +qzmfV6GnRtuUrBLVJ8qyjif1JSM9tZfinnmAB4/U5Qfc+YYViIXMTljmHWvbokas +tTBfVAw74yWnkv4ZuXf5SkTmGwEMJUOat0TSr085Ck5y394bRepdI1Y+1cdqpwMQ +QmkKyvcBlREQ7Xk1UnDDR3o/2ieVuGGHRp8jmoWBWGq4Cm43fYOlVe+PcaX0tDns +Tmmh2uwEU/TXe5qGil51OlSM7qhAMqhWUIYphSOcdvApNXuiWMfnTdjsNygE4HVh +Jq4efJ/nlx5N+PNAK2GpzeUJQGyxiVsXybq+h8UlvytBsdz1X6ZYzBv1yYwANThU +rMB1s4tMaEugX0aNByLcsxuS4ixd2qzwkYVz25Aeko/U1v2/j2cIRtrTNgja3BKE +N5Ug +=80Es +-----END PGP PUBLIC KEY BLOCK----- diff --git a/contrib/gitian-keys/willyko-key.pgp b/contrib/gitian-keys/willyko-key.pgp new file mode 100644 index 0000000000..a3b58ad994 --- /dev/null +++ b/contrib/gitian-keys/willyko-key.pgp @@ -0,0 +1,85 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQINBFgs/RoBEADFxycJTUvwqzBZZ0aBZXbmr8Ppd3EPrgBRd47k7uwanf7UFmvY +Xt4gMEI+EdV0GuoQ0SeoAmQqc5Fxu3AQe2XFbiF+ZNNYT3+V/5GAzWsAH22ncQr0 +AuK95pPi+PZ+M2h669cq/RzFUXZDew0NobR2oBS5h6g3rgmmejVLRqnUpWkkSrqi +aNgD2GSn8g820wM6LpdxcjTqmMpHHT5owAbv0UP3IcdtpBaS5McoUXK+OAdKK/Zw +JQ0J1kx4vIyNwuPD3klziGQw8Izb/gFpWg8XaJmMhD5BxNuXJC58Bj9/sFTc0GDQ +VKMFpYpNi8a6hLPFb4hMjYF77awoz57HtyOOsS03KO/57QE1htx+2NeDm4XkZSBk ++wrU3zgbtmOBcfzEHS/HrROksYDi+Qw3HZL98nfDEWNfsDzfhMZ9wHdM3NsR2xk6 +oNtX0CprS1n2Xr2AY9X1oNgiZCJaSftU67j3lr+9gHOH61ktxt3cUCDodUFjkpKn +r1CQ2LB63AoUbwGMAeozdXZWzbXJAJbcH9G77zEi9rW0WA2yMSxTXHlpE9MS0UcE +BVkIMv2b9iQzlhiS8jh8AiKFO1PuT26Cw52N/lSPhA81zw79pZfSYwKFICGHYfvw +ozZeN9Q+PPl5tqi/3SExxlZKe8EmaveTrUfKMBS4lQO2gWe0bCFgLOIzIwARAQAB +tB1XaWxseSBLbyA8d2lsbHlrQHN5c2NvaW4ub3JnPokCOAQTAQIAIgUCWCz9GgIb +AwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQjjqPMkfby7+0wA//cX7Tc3Nz +19ApwSbGfC8pJA/nSybcVivroJRftpzeOmYrVM084T9REvYwugl89djvxn6m96iQ +kqoUGWhBVBtDReVCL7z53G42lHjemaFcxBhIazKxO0qvcc/UXUVOs2OdUbzObDFL +dHO5xBVqEnW3sq+r4blsXR8U79B9IIri4+2hy4OoEjYv9DzBaaoaqU+F3mudXbmo +R+hsWc+mklV++TX/kuw6EWT8tusFjXrfqqKcKPRPhbn48OSGWsEPc7yELf7pYFR8 +uDU40faJqkvQ83h5WMTDAhLxd/918ZitqBhjSP+7Humf2YhSto7YmtEWlbeAW+qy +TcBYkK6SJh8Do3xZd/prFBKEu395n5VQKuLjXaOjqMc1oDHQyPJJjXSN5thLHvan +z7nNLt2QZO/kxXITDdbWlktVe/WSoive7TuY4dGuX4Si2z9wyhFYxtZDsqE0qmqN +jIDAZ7u8Qq/LGqpdjOmYr2fEwHe1yVIS+BtVGvtShkX+J+QPb8qBl1d7Ii5i5Afl +GJoLLIUFkPcIRTYPZpppGSuqfyWdNnaasbLH44lxJisSMMw+fxZabt2bykYN/ZXa +RP/ItDj81vklg+n6r4f/nZTF1r0UUy4LbSbBY15B4Xm0Tdvh1PMfj/w2q10l7bZB +XLi9Z/QPaW7TyzaBuLkVckbVFn2nYnXfzHG5Ag0EWCz9GgEQALCgTibFnw+Q3PEL +G5/peQcQqHxrPAB37HV39B1DedGhVUa6aGSUaLoNMyUjUX1HWN3mWFKTYVB4CH5Y +xjaXUwxdwCZgBNe4TDglKFPuc+frlSTZxDVE9/fjArmrUP6TPU447ujspyngGLa5 +et5Uig/LxIX/+Mm0ZiYJxb1rMJwK998U1Ev1aHxgNjwTI2ehcKu8CAGOyflzh6a2 +iTBUmLfnQMv5248P2d4P8WDiPq61CWTYTMCFqHqkYKy7h9BYIuMajw3KsgOUNfL2 +1e9Ue8yv5UchZ+GDlBjidIkoK1nd2uJ0kPJkafLGWbcliJfvXxKliZnSbz1Cd4A0 +HDKKCwiuwSLy2aYbs7IRtAOyDER4+fjBcqtf0QTIvoAdNZ9gL64DKVaB58vuSixj +K1i83XbTOt3q821HxxBrX9u6HP2E5kFdxT2KHDbisAWNP0rFnHVpjugehKFfZb6q +jbDt3nQL5uCQ8gTNCd4fsoSK6KhCDjamDXlKmaGlxqwOV4W8ZwihoeGt690h7NIH +h4eiSmMOej3or32lcDETEwrjA2PxvcFsikFc56hRkTaSyyBEH2xhkRrjXMqiQfH0 +j7iOY2PWpFEuu2HVzqe5dBXzn9sMIwxeNCxR/P+xHMqPUlgD1SXEYCNLvvzD6p0+ +kqSe7PiJoEIv351T3hnBhQ6rK0ChABEBAAGJAh8EGAECAAkFAlgs/RoCGwwACgkQ +jjqPMkfby7/mQA//YsAOdDBl0GscB1PBNXi8VMlI7yG9cqiGrYnZX7h4wUoGEbPI +jap/PixIsxBCf1BqBRDJdFyvzH9amLlcaVNdCyh6Yt1Pi8kassmz/kbIYgpbFkIL +ES9N24N7BZ94P77OQy5wic+B4WqJnVrtKr9JBalgBSOMqtccYCma5Ew00mqp+FXM +suDyBk2HXyl+u6/rRmqZ+BoU8iRpus9F80LFKGEsAgjLjKv68KmApzjunzsBotKk +g9AsBk4ygbp+nECAtsxpbLMo4hPr4qWm7G4mU5g4xOK2chpAPeqyf0857RWgsXaO +kjrUu/M5Hme2eIlXwBF14ac4QPnY1rlAIaulvXzmQnMYQFZiw9vaTOdqBFHjkh7T +XYRAr589Woo25PfMJCbC+Rop6ku6sCFMorbBwojyRhFJnk9xsy5kP5D9IhkPAKu/ +/ABlei0xPOl/gCUUJP7aIikZgS5lAk1TSe/R+yV6ExNwudtLw1G0K2/sY3B4Xo3X +Q1lTAQPlnAIeK/vlbttLZNIBWquw4cPAkPpIyjmE1dd6jGQdUyZE22uPBx+gpq1w +AacmVLwvPMe1De0ilJOzj6KpXWBCwt0DWXWztovpBVcAC+qbTrZF9H5dllpqyzKt +OvxzGssjrX4rDkOx7MyVa2tnXmeCuSN/RvlOUwPvf5zYM8Wh9g7fc6jcDQu5Ag0E +WfkOfAEQAMNkzAQqSenpXtHsnuCqM1oMMF2kRzny/Jqh3q3BxZ8MHLDhoRRaTENu +lA4APRXMNM/wlZJUSLX8wWBhufnsPtMf6OOVMZ4AVbXHjUgyJ7lO1zHdj0u6PpYP +9gmHthIz7FF+cxHj4ziC4CmtRctrn+/U4MwYtNPhxkTnS26oOZes/HXMYSvQBMgT +AP27GNOBiJRthjIEITvSvS0YZOxgLtWgGiks/pGUw5wm1rguuQVyZ1/LfXBooYJo +u/v21AEjpuTg7JlwbqXr2k5LojAGq7AxDyWy21IW0E45Gog38zg/hwNll+hjRbSu +pipf74WXR6xMMlW6A+XWUvElkicfDx8e9LJUnqWbZ+FL7X4SB54ZHNCvfo/8Ug1V +2tiY9WbUZL9n5ZQHNlk3J+UK/KDvwey1VzKPFjpQNlfahhnppDGiCey+mERjI+75 +gPbk0ctOAEYXgLJjoonGX+iByAfY0YyJF281CtaK/sXQU+TzLLT15WET+gYGsJdY +xh1PdPscNdSgYudvbKZoFnqUwEGEfD8dT5bjOphfY5+LvGUR2GuLNZpMidcduTYf +SWAY/vQHQIJArXu29BKscm3tg6tzXu3l9p/bGIQUQB7obN91y3xD3BLICIPRGhKE +924wxxCuH1vLKmxWDdAAxKo+rEdLJ/rbZnjWQENEFiJ114fBk2NVABEBAAGJBEQE +GAEIAA8FAln5DnwCGwIFCQPCZwACKQkQjjqPMkfby7/BXSAEGQEIAAYFAln5DnwA +CgkQYFGSo/6YSmANoQ//SbcKxkop2zA2HrWS4THcEJQwSJ0KGAN/VB83JQhoWThX +CWxsFNJjBy7+rsoXd3wQG1/aN42nTuj+eh+R6WJJaqqnMqd52l4Kc1kJA6z4DGsy +3azCDvyzibM0AkJyMJyYi6HRKjzA4M+xKR1HoT/NdQUP5CBUVfvMblSaOWiw4rja +IhWcbgbQ+Zam/VaV5l1O90eaD9tL3twSfPLYZ/wkeO63jJKHBpI8fpMql/bLg9WD +Au3h/lU63NWe5lZO1z/jIdfiTSvg8nu162vcOgmUCWo9spkybjJd0Mx6ZId79rVo +58lwZ4QoaMgPGoVP67LyLOxJTIXeyG5xr1LxhMPMGbnBhlnMQrboLV9kPEL3raHE +EEKDTtZimVK3ZxmfyBd6MDmwcL/K73xu/R8be9TgdwD8/BZJSOTkO87qZ82G9T7E +oY5IHU+qd41/Yjbut7AVtAlCr5Lor31EYvZh3gI/H8uZFddOu37Ij7e9Fw2ywv3A +wPks89tfOvahkfCOJ29znB+uQYpJ461jjhdkB7EHG4ae07M5rRtkNbIc3dqbnMhz +VA3JpRJN77xPXV7uITHo1s+b50RvWmfYW91zvipaSZxbMLuGBMhn/1QaM1djLOYN +JordDBwEr2bi5a063yUbZrk6ddECuyxndDHWDNr+Tqx6o7lmAT48UJ199zA4scbf +2g/9EiRPGcRovsn1tUdjzfmWDxhrRV6F3rYJB1+i6Mqeg2iHHYxxiNDXcuWYXHQ/ +WPWLk5+lgh0rQbrE7InzEejoM0FIHzLTm0lSQpau50/PT2FiH6sOEEDyT2IhBtXX +eOnKAi1IfGNMzEaPEY8PXH78dEGv0iXIgy4l8Bc57q09Z9R/OUi+Yb1p+S5F/aOi +7Jd53GGE1bfBIlsMos092XoiMdvKmAczyCUIempKHUBPoqfJge77qk7zJKkyM3Dk +VX0lXLdhj0PfslFrNf2uRF4uZkmfUV7peeD023c0/SVp3ILUAVds52yawi6Exv4a +bbvhIw72fc31frCRBqc9HVsBraoozzE9bksG1MdNI3GgKxecOu9lldedlIqi4lO4 +7kTVDLEmcsQO+sSxkXQz2sMSD01CQndpPuhFNlqvVnfK+Kv8pSG37VzSSQz1nt5K +w/fJBo4T/ztR7D9RzbSDxBP8Jjaa+UYabjab5HcE0JI4CpgmzIOB7qPVbYCn+LNX +c8Xw5/9iTw+ayawl7PCGRfd14/OPRzI8vS0I9bF8AG84XM46yxAtYieH/9RI3b6/ +GiQYDkBNi6Kb1LfSzx8oKAkbMgiy4y3vWxLQnE34bAoXjGiYdAMliOsyGcvmnObD +GmSTIlIqunq60CyhaUSIkl2VRhjzz0igfS9751XEvnjeXDc= +=PVBi +-----END PGP PUBLIC KEY BLOCK----- diff --git a/contrib/init/bitcoind.openrc b/contrib/init/bitcoind.openrc index 50377c0995..86222295db 100644 --- a/contrib/init/bitcoind.openrc +++ b/contrib/init/bitcoind.openrc @@ -1,4 +1,4 @@ -#!/sbin/runscript +#!/sbin/openrc-run # backward compatibility for existing gentoo layout # diff --git a/contrib/init/bitcoind.service b/contrib/init/bitcoind.service index 9132957c38..ee113d7615 100644 --- a/contrib/init/bitcoind.service +++ b/contrib/init/bitcoind.service @@ -1,22 +1,25 @@ +# It is not recommended to modify this file in-place, because it will +# be overwritten during package upgrades. If you want to add further +# options or overwrite existing ones then use +# $ systemctl edit bitcoind.service +# See "man systemd.service" for details. + +# Note that almost all daemon options could be specified in +# /etc/bitcoin/bitcoin.conf + [Unit] -Description=Bitcoin's distributed currency daemon +Description=Bitcoin daemon After=network.target [Service] +ExecStart=/usr/bin/bitcoind -daemon -conf=/etc/bitcoin/bitcoin.conf -pid=/run/bitcoind/bitcoind.pid +# Creates /run/bitcoind owned by bitcoin +RuntimeDirectory=bitcoind User=bitcoin -Group=bitcoin - Type=forking -PIDFile=/var/lib/bitcoind/bitcoind.pid -ExecStart=/usr/bin/bitcoind -daemon -pid=/var/lib/bitcoind/bitcoind.pid \ --conf=/etc/bitcoin/bitcoin.conf -datadir=/var/lib/bitcoind -disablewallet - -Restart=always +PIDFile=/run/bitcoind/bitcoind.pid +Restart=on-failure PrivateTmp=true -TimeoutStopSec=60s -TimeoutStartSec=2s -StartLimitInterval=120s -StartLimitBurst=5 [Install] WantedBy=multi-user.target diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh new file mode 100755 index 0000000000..b57ade139d --- /dev/null +++ b/contrib/install_db4.sh @@ -0,0 +1,86 @@ +#!/bin/sh + +# Install libdb4.8 (Berkeley DB). + +set -e + +if [ -z "${1}" ]; then + echo "Usage: ./install_db4.sh <base-dir> [<extra-bdb-configure-flag> ...]" + echo + echo "Must specify a single argument: the directory in which db4 will be built." + echo "This is probably \`pwd\` if you're at the root of the bitcoin repository." + exit 1 +fi + +expand_path() { + echo "$(cd "${1}" && pwd -P)" +} + +BDB_PREFIX="$(expand_path ${1})/db4"; shift; +BDB_VERSION='db-4.8.30.NC' +BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' +BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" + +check_exists() { + which "$1" >/dev/null 2>&1 +} + +sha256_check() { + # Args: <sha256_hash> <filename> + # + if check_exists sha256sum; then + echo "${1} ${2}" | sha256sum -c + elif check_exists sha256; then + if [ "$(uname)" = "FreeBSD" ]; then + sha256 -c "${1}" "${2}" + else + echo "${1} ${2}" | sha256 -c + fi + else + echo "${1} ${2}" | shasum -a 256 -c + fi +} + +http_get() { + # Args: <url> <filename> <sha256_hash> + # + # It's acceptable that we don't require SSL here because we manually verify + # content hashes below. + # + if [ -f "${2}" ]; then + echo "File ${2} already exists; not downloading again" + elif check_exists curl; then + curl --insecure "${1}" -o "${2}" + else + wget --no-check-certificate "${1}" -O "${2}" + fi + + sha256_check "${3}" "${2}" +} + +mkdir -p "${BDB_PREFIX}" +http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" +tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" +cd "${BDB_PREFIX}/${BDB_VERSION}/" + +# Apply a patch necessary when building with clang and c++11 (see https://community.oracle.com/thread/3952592) +CLANG_CXX11_PATCH_URL='https://gist.githubusercontent.com/LnL7/5153b251fd525fe15de69b67e63a6075/raw/7778e9364679093a32dec2908656738e16b6bdcb/clang.patch' +CLANG_CXX11_PATCH_HASH='7a9a47b03fd5fb93a16ef42235fa9512db9b0829cfc3bdf90edd3ec1f44d637c' +http_get "${CLANG_CXX11_PATCH_URL}" clang.patch "${CLANG_CXX11_PATCH_HASH}" +patch -p2 < clang.patch + +cd build_unix/ + +"${BDB_PREFIX}/${BDB_VERSION}/dist/configure" \ + --enable-cxx --disable-shared --with-pic --prefix="${BDB_PREFIX}" \ + "${@}" + +make install + +echo +echo "db4 build complete." +echo +echo 'When compiling bitcoind, run `./configure` in the following way:' +echo +echo " export BDB_PREFIX='${BDB_PREFIX}'" +echo ' ./configure LDFLAGS="-L${BDB_PREFIX}/lib/" CPPFLAGS="-I${BDB_PREFIX}/include/" ...' diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index afcec2b60a..4969e96827 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -2,7 +2,7 @@ # # linearize-data.py: Construct a linear, no-fork version of the chain. # -# Copyright (c) 2013-2016 The Bitcoin Core developers +# Copyright (c) 2013-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 58fec6dddc..6b69c5b3a3 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -2,7 +2,7 @@ # # linearize-hashes.py: List blocks in a linear, no-fork version of the chain. # -# Copyright (c) 2013-2016 The Bitcoin Core developers +# Copyright (c) 2013-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 7f017bb4f1..3379a4599c 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -40,7 +40,7 @@ grep CodeResources < "${TEMPLIST}" | while read i; do RESOURCE="${TEMPDIR}/${OUTROOT}/${TARGETFILE}" DIRNAME="`dirname "${RESOURCE}"`" mkdir -p "${DIRNAME}" - echo "Adding resource for: "${TARGETFILE}"" + echo "Adding resource for: \"${TARGETFILE}\"" cp "${i}" "${RESOURCE}" done diff --git a/contrib/testgen/base58.py b/contrib/testgen/base58.py index 0d09692b36..816d40b49c 100644 --- a/contrib/testgen/base58.py +++ b/contrib/testgen/base58.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012-2016 The Bitcoin Core developers +# Copyright (c) 2012-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/testgen/gen_base58_test_vectors.py b/contrib/testgen/gen_base58_test_vectors.py index 86366eb8e3..8e6a5d5819 100755 --- a/contrib/testgen/gen_base58_test_vectors.py +++ b/contrib/testgen/gen_base58_test_vectors.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2012-2016 The Bitcoin Core developers +# Copyright (c) 2012-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/contrib/tidy_datadir.sh b/contrib/tidy_datadir.sh index 8960f8811d..b845b34e41 100755 --- a/contrib/tidy_datadir.sh +++ b/contrib/tidy_datadir.sh @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. if [ -d "$1" ]; then - cd "$1" + cd "$1" || exit 1 else echo "Usage: $0 <datadir>" >&2 echo "Removes obsolete Bitcoin database files" >&2 diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp new file mode 100644 index 0000000000..0f6d993fd2 --- /dev/null +++ b/contrib/valgrind.supp @@ -0,0 +1,43 @@ +# Valgrind suppressions file for Bitcoin. +# +# Includes known Valgrind warnings in our dependencies that cannot be fixed +# in-tree. +# +# Example use: +# $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin +# $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ +# --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite +{ + Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + obj:*/libstdc++.* + fun:call_init.part.0 + fun:call_init + fun:_dl_init + obj:*/ld-*.so +} +{ + Suppress libdb warning - https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=662917 + Memcheck:Cond + obj:*/libdb_cxx-*.so + fun:__log_put + obj:*/libdb_cxx-*.so + fun:__log_put_record +} +{ + Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7leveldbL10InitModuleEv +} +{ + Suppress leveldb warning (leveldb::Env::Default()) - https://github.com/google/leveldb/issues/113 + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_ZN7leveldbL14InitDefaultEnvEv +} diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index b01e2a6d39..8f3e4b8063 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -9,7 +9,7 @@ REVSIG=false IFS=' ' if [ "$BITCOIN_VERIFY_COMMITS_ALLOW_SHA1" = 1 ]; then - GPG_RES="$(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" else # Note how we've disabled SHA1 with the --weak-digest option, disabling # signatures - including selfsigs - that use SHA1. While you might think that @@ -24,7 +24,7 @@ else case "$LINE" in "gpg (GnuPG) 1.4.1"*|"gpg (GnuPG) 2.0."*) echo "Please upgrade to at least gpg 2.1.10 to check for weak signatures" > /dev/stderr - GPG_RES="$(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" ;; # We assume if you're running 2.1+, you're probably running 2.1.10+ # gpg will fail otherwise @@ -32,7 +32,7 @@ else # gpg will fail otherwise esac done - [ "$GPG_RES" = "" ] && GPG_RES="$(echo "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)" + [ "$GPG_RES" = "" ] && GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)" fi for LINE in $(echo "$GPG_RES"); do case "$LINE" in @@ -46,14 +46,19 @@ for LINE in $(echo "$GPG_RES"); do REVSIG=true GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" ;; + "[GNUPG:] EXPKEYSIG "*) + [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 + REVSIG=true + GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" + ;; esac done if ! $VALID; then exit 1 fi if $VALID && $REVSIG; then - echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" echo "$GOODREVSIG" else - echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null fi diff --git a/contrib/verify-commits/verify-commits.sh b/contrib/verify-commits/verify-commits.sh index 74b7f38375..532b97a438 100755 --- a/contrib/verify-commits/verify-commits.sh +++ b/contrib/verify-commits/verify-commits.sh @@ -12,8 +12,6 @@ VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root") VERIFIED_SHA512_ROOT=$(cat "${DIR}/trusted-sha512-root-commit") REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits") -HAVE_FAILED=false - HAVE_GNU_SHA512=1 [ ! -x "$(which sha512sum)" ] && HAVE_GNU_SHA512=0 @@ -35,11 +33,12 @@ fi NO_SHA1=1 PREV_COMMIT="" +INITIAL_COMMIT="${CURRENT_COMMIT}" while true; do if [ "$CURRENT_COMMIT" = $VERIFIED_ROOT ]; then - echo "There is a valid path from "$CURRENT_COMMIT" to $VERIFIED_ROOT where all commits are signed!" - exit 0; + echo "There is a valid path from \"$INITIAL_COMMIT\" to $VERIFIED_ROOT where all commits are signed!" + exit 0 fi if [ "$CURRENT_COMMIT" = $VERIFIED_SHA512_ROOT ]; then @@ -95,9 +94,9 @@ while true; do FILE_HASHES="" for FILE in $(git ls-tree --full-tree -r --name-only "$CURRENT_COMMIT" | LC_ALL=C sort); do if [ "$HAVE_GNU_SHA512" = 1 ]; then - HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | sha512sum | { read FIRST OTHER; echo $FIRST; } ) + HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | sha512sum | { read FIRST _; echo $FIRST; } ) else - HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | shasum -a 512 | { read FIRST OTHER; echo $FIRST; } ) + HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | shasum -a 512 | { read FIRST _; echo $FIRST; } ) fi [ "$FILE_HASHES" != "" ] && FILE_HASHES="$FILE_HASHES"' ' diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh index 409f517c9f..e0266bf08a 100755 --- a/contrib/verifybinaries/verify.sh +++ b/contrib/verifybinaries/verify.sh @@ -33,7 +33,7 @@ if [ ! -d "$WORKINGDIR" ]; then mkdir "$WORKINGDIR" fi -cd "$WORKINGDIR" +cd "$WORKINGDIR" || exit 1 #test if a version number has been passed as an argument if [ -n "$1" ]; then @@ -76,8 +76,6 @@ if [ -n "$1" ]; then BASEDIR="$BASEDIR$RCSUBDIR.$RCVERSION/" fi fi - - SIGNATUREFILE="$BASEDIR$SIGNATUREFILENAME" else echo "Error: need to specify a version on the command line" exit 2 @@ -89,7 +87,7 @@ WGETOUT=$(wget -N "$HOST1$BASEDIR$SIGNATUREFILENAME" 2>&1) #and then see if wget completed successfully if [ $? -ne 0 ]; then echo "Error: couldn't fetch signature file. Have you specified the version number in the following format?" - echo "[$VERSIONPREFIX]<version>-[$RCVERSIONSTRING[0-9]] (example: "$VERSIONPREFIX"0.10.4-"$RCVERSIONSTRING"1)" + echo "[$VERSIONPREFIX]<version>-[$RCVERSIONSTRING[0-9]] (example: ${VERSIONPREFIX}0.10.4-${RCVERSIONSTRING}1)" echo "wget output:" echo "$WGETOUT"|sed 's/^/\t/g' exit 2 diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py index 5cc19761d4..6e44c56f52 100755 --- a/contrib/zmq/zmq_sub.py +++ b/contrib/zmq/zmq_sub.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ Bitcoin should be started with the command line arguments: bitcoind -testnet -daemon \ - -zmqpubhashblock=tcp://127.0.0.1:28332 \ -zmqpubrawtx=tcp://127.0.0.1:28332 \ + -zmqpubrawblock=tcp://127.0.0.1:28332 \ -zmqpubhashtx=tcp://127.0.0.1:28332 \ -zmqpubhashblock=tcp://127.0.0.1:28332 diff --git a/contrib/zmq/zmq_sub3.4.py b/contrib/zmq/zmq_sub3.4.py index bfb7ea9eae..536352d5ff 100755 --- a/contrib/zmq/zmq_sub3.4.py +++ b/contrib/zmq/zmq_sub3.4.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ Bitcoin should be started with the command line arguments: bitcoind -testnet -daemon \ - -zmqpubhashblock=tcp://127.0.0.1:28332 \ -zmqpubrawtx=tcp://127.0.0.1:28332 \ + -zmqpubrawblock=tcp://127.0.0.1:28332 \ -zmqpubhashtx=tcp://127.0.0.1:28332 \ -zmqpubhashblock=tcp://127.0.0.1:28332 diff --git a/depends/README.md b/depends/README.md index 6053c531b4..99eef1952c 100644 --- a/depends/README.md +++ b/depends/README.md @@ -28,6 +28,22 @@ Common `host-platform-triplets` for cross compilation are: No other options are needed, the paths are automatically configured. +Install the required dependencies: Ubuntu & Debian +-------------------------------------------------- + +For macOS cross compilation: + + sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python-setuptools + +For Win32/Win64 cross compilation: + +- see [build-windows.md](../doc/build-windows.md#cross-compilation-for-ubuntu-and-windows-subsystem-for-linux) + +For linux (including i386, ARM) cross compilation: + + sudo apt-get install curl g++-aarch64-linux-gnu g++-4.8-aarch64-linux-gnu gcc-4.8-aarch64-linux-gnu binutils-aarch64-linux-gnu g++-arm-linux-gnueabihf g++-4.8-arm-linux-gnueabihf gcc-4.8-arm-linux-gnueabihf binutils-arm-linux-gnueabihf g++-4.8-multilib gcc-4.8-multilib binutils-gold bsdmainutils + + Dependency Options: The following can be set when running make: make FOO=bar diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 01146c26f6..3f97221e10 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -1,19 +1,18 @@ package=zeromq -$(package)_version=4.1.5 -$(package)_download_path=https://github.com/zeromq/zeromq4-1/releases/download/v$($(package)_version)/ +$(package)_version=4.2.2 +$(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=04aac57f081ffa3a2ee5ed04887be9e205df3a7ddade0027460b8042432bdbcf -$(package)_patches=9114d3957725acd34aa8b8d011585812f3369411.patch 9e6745c12e0b100cd38acecc16ce7db02905e27c.patch +$(package)_sha256_hash=5b23f4ca9ef545d5bd3af55d305765e3ee06b986263b31967435d285a3e6df6b +$(package)_patches=0001-fix-build-with-older-mingw64.patch define $(package)_set_vars - $(package)_config_opts=--without-documentation --disable-shared --without-libsodium --disable-curve + $(package)_config_opts=--without-docs --disable-shared --without-libsodium --disable-curve --disable-curve-keygen --disable-perf $(package)_config_opts_linux=--with-pic $(package)_cxxflags=-std=c++11 endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/9114d3957725acd34aa8b8d011585812f3369411.patch && \ - patch -p1 < $($(package)_patch_dir)/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch && \ + patch -p1 < $($(package)_patch_dir)/0001-fix-build-with-older-mingw64.patch && \ ./autogen.sh endef @@ -22,7 +21,7 @@ define $(package)_config_cmds endef define $(package)_build_cmds - $(MAKE) libzmq.la + $(MAKE) src/libzmq.la endef define $(package)_stage_cmds @@ -30,5 +29,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds + sed -i.old "s/ -lstdc++//" lib/pkgconfig/libzmq.pc && \ rm -rf bin share endef diff --git a/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch b/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch new file mode 100644 index 0000000000..a6c508fb8a --- /dev/null +++ b/depends/patches/zeromq/0001-fix-build-with-older-mingw64.patch @@ -0,0 +1,30 @@ +From 1a159c128c69a42d90819375c06a39994f3fbfc1 Mon Sep 17 00:00:00 2001 +From: Cory Fields <cory-nospam-@coryfields.com> +Date: Tue, 28 Nov 2017 20:33:25 -0500 +Subject: [PATCH] fix build with older mingw64 + +--- + src/windows.hpp | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/src/windows.hpp b/src/windows.hpp +index 99e889d..e69038e 100644 +--- a/src/windows.hpp ++++ b/src/windows.hpp +@@ -55,6 +55,13 @@ + #include <winsock2.h> + #include <windows.h> + #include <mswsock.h> ++ ++#if defined __MINGW64_VERSION_MAJOR && __MINGW64_VERSION_MAJOR < 4 ++// Workaround for mingw-w64 < v4.0 which did not include ws2ipdef.h in iphlpapi.h. ++// Fixed in mingw-w64 by 9bd8fe9148924840d315b4c915dd099955ea89d1. ++#include <ws2def.h> ++#include <ws2ipdef.h> ++#endif + #include <iphlpapi.h> + + #if !defined __MINGW32__ +-- +2.7.4 + diff --git a/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch b/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch deleted file mode 100644 index f704b3d94f..0000000000 --- a/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 9114d3957725acd34aa8b8d011585812f3369411 Mon Sep 17 00:00:00 2001 -From: Jeroen Ooms <jeroenooms@gmail.com> -Date: Tue, 20 Oct 2015 13:10:38 +0200 -Subject: [PATCH] enable static libraries on mingw - ---- - configure.ac | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 393505b..e92131a 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -265,7 +265,7 @@ case "${host_os}" in - libzmq_dso_visibility="no" - - if test "x$enable_static" = "xyes"; then -- AC_MSG_ERROR([Building static libraries is not supported under MinGW32]) -+ CPPFLAGS="-DZMQ_STATIC" - fi - - # Set FD_SETSIZE to 1024
\ No newline at end of file diff --git a/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch b/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch deleted file mode 100644 index 9aff2c179a..0000000000 --- a/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 9e6745c12e0b100cd38acecc16ce7db02905e27c Mon Sep 17 00:00:00 2001 -From: David Millard <dmillard10@gmail.com> -Date: Tue, 10 May 2016 13:53:53 -0700 -Subject: [PATCH] Fix autotools for static MinGW builds - ---- - configure.ac | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 5a0fa14..def6ea7 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -259,7 +259,7 @@ case "${host_os}" in - libzmq_dso_visibility="no" - - if test "x$enable_static" = "xyes"; then -- CPPFLAGS="-DZMQ_STATIC" -+ CPPFLAGS="-DZMQ_STATIC $CPPFLAGS" - fi - - # Set FD_SETSIZE to 1024
\ No newline at end of file diff --git a/doc/bips.md b/doc/bips.md index bc8dcb6fb3..e587275f0f 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -33,3 +33,5 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.13.0**): * [`BIP 145`](https://github.com/bitcoin/bips/blob/master/bip-0145.mediawiki): getblocktemplate updates for Segregated Witness as of **v0.13.0** ([PR 8149](https://github.com/bitcoin/bitcoin/pull/8149)). * [`BIP 147`](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki): NULLDUMMY softfork as of **v0.13.1** ([PR 8636](https://github.com/bitcoin/bitcoin/pull/8636) and [PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)). * [`BIP 152`](https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki): Compact block transfer and related optimizations are used as of **v0.13.0** ([PR 8068](https://github.com/bitcoin/bitcoin/pull/8068)). +* [`BIP 159`](https://github.com/bitcoin/bips/blob/master/bip-0159.mediawiki): NODE_NETWORK_LIMITED service bit [signaling only] is supported as of **v0.16.0** ([PR 11740](https://github.com/bitcoin/bitcoin/pull/11740)). +* [`BIP 176`](https://github.com/bitcoin/bips/blob/master/bip-0176.mediawiki): Bits Denomination [QT only] is supported as of **v0.16.0** ([PR 12035](https://github.com/bitcoin/bitcoin/pull/12035)). diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 4ea8e30b53..0817821221 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,10 +1,10 @@ OpenBSD build guide ====================== -(updated for OpenBSD 6.1) +(updated for OpenBSD 6.2) This guide describes how to build bitcoind and command-line utilities on OpenBSD. -As OpenBSD is most common as a server OS, we will not bother with the GUI. +OpenBSD is most commonly used as a server OS, so this guide does not contain instructions for building the GUI. Preparation ------------- @@ -12,101 +12,42 @@ Preparation Run the following as root to install the base dependencies for building: ```bash -pkg_add gmake libtool libevent +pkg_add git gmake libevent libtool pkg_add autoconf # (select highest version, e.g. 2.69) pkg_add automake # (select highest version, e.g. 1.15) -pkg_add python # (select highest version, e.g. 3.5) -``` - -See [dependencies.md](dependencies.md) for a complete overview. - -GCC -------- - -The default C++ compiler that comes with OpenBSD 5.9 is g++ 4.2. This version is old (from 2007), and is not able to compile the current version of Bitcoin Core, primarily as it has no C++11 support, but even before there were issues. So here we will be installing a newer compiler: +pkg_add python # (select highest version, e.g. 3.6) +pkg_add boost -```bash -pkg_add g++ # (select newest 4.x version, e.g. 4.9.3) +git clone https://github.com/bitcoin/bitcoin.git ``` -This compiler will not overwrite the system compiler, it will be installed as `egcc` and `eg++` in `/usr/local/bin`. - -### Building boost - -Do not use `pkg_add boost`! The boost version installed thus is compiled using the `g++` compiler not `eg++`, which will result in a conflict between `/usr/local/lib/libestdc++.so.XX.0` and `/usr/lib/libstdc++.so.XX.0`, resulting in a test crash: - - test_bitcoin:/usr/lib/libstdc++.so.57.0: /usr/local/lib/libestdc++.so.17.0 : WARNING: symbol(_ZN11__gnu_debug17_S_debug_me ssagesE) size mismatch, relink your program - ... - Segmentation fault (core dumped) - -This makes it necessary to build boost, or at least the parts used by Bitcoin Core, manually: +See [dependencies.md](dependencies.md) for a complete overview. -``` -# Pick some path to install boost to, here we create a directory within the bitcoin directory -BITCOIN_ROOT=$(pwd) -BOOST_PREFIX="${BITCOIN_ROOT}/boost" -mkdir -p $BOOST_PREFIX - -# Fetch the source and verify that it is not tampered with -curl -o boost_1_64_0.tar.bz2 https://netcologne.dl.sourceforge.net/project/boost/boost/1.64.0/boost_1_64_0.tar.bz2 -echo '7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 boost_1_64_0.tar.bz2' | sha256 -c -# MUST output: (SHA256) boost_1_64_0.tar.bz2: OK -tar -xjf boost_1_64_0.tar.bz2 - -# Boost 1.64 needs one small patch for OpenBSD -cd boost_1_64_0 -# Also here: https://gist.githubusercontent.com/laanwj/bf359281dc319b8ff2e1/raw/92250de8404b97bb99d72ab898f4a8cb35ae1ea3/patch-boost_test_impl_execution_monitor_ipp.patch -patch -p0 < /usr/ports/devel/boost/patches/patch-boost_test_impl_execution_monitor_ipp - -# Build w/ minimum configuration necessary for bitcoin -echo 'using gcc : : eg++ : <cxxflags>"-fvisibility=hidden -fPIC" <linkflags>"" <archiver>"ar" <striper>"strip" <ranlib>"ranlib" <rc>"" : ;' > user-config.jam -config_opts="runtime-link=shared threadapi=pthread threading=multi link=static variant=release --layout=tagged --build-type=complete --user-config=user-config.jam -sNO_BZIP2=1" -./bootstrap.sh --without-icu --with-libraries=chrono,filesystem,program_options,system,thread,test -./b2 -d2 -j2 -d1 ${config_opts} --prefix=${BOOST_PREFIX} stage -./b2 -d0 -j4 ${config_opts} --prefix=${BOOST_PREFIX} install -``` +**Important**: From OpenBSD 6.2 onwards a C++11-supporting clang compiler is +part of the base image, and while building it is necessary to make sure that this +compiler is used and not ancient g++ 4.2.1. This is done by appending +`CC=cc CXX=c++` to configuration commands. Mixing different compilers +within the same executable will result in linker errors. ### Building BerkeleyDB -BerkeleyDB is only necessary for the wallet functionality. To skip this, pass `--disable-wallet` to `./configure`. +BerkeleyDB is only necessary for the wallet functionality. To skip this, pass +`--disable-wallet` to `./configure` and skip to the next section. -See "Berkeley DB" in [build_unix.md](build_unix.md) for instructions on how to build BerkeleyDB 4.8. -You cannot use the BerkeleyDB library from ports, for the same reason as boost above (g++/libstd++ incompatibility). +It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library +from ports, for the same reason as boost above (g++/libstd++ incompatibility). +If you have to build it yourself, you can use [the installation script included +in contrib/](/contrib/install_db4.sh) like so -```bash -# Pick some path to install BDB to, here we create a directory within the bitcoin directory -BITCOIN_ROOT=$(pwd) -BDB_PREFIX="${BITCOIN_ROOT}/db4" -mkdir -p $BDB_PREFIX - -# Fetch the source and verify that it is not tampered with -curl -o db-4.8.30.NC.tar.gz 'http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz' -echo '12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef db-4.8.30.NC.tar.gz' | sha256 -c -# MUST output: (SHA256) db-4.8.30.NC.tar.gz: OK -tar -xzf db-4.8.30.NC.tar.gz - -# Build the library and install to specified prefix -cd db-4.8.30.NC/build_unix/ -# Note: Do a static build so that it can be embedded into the executable, instead of having to find a .so at runtime -../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX CC=egcc CXX=eg++ CPP=ecpp -make install # do NOT use -jX, this is broken +```shell +./contrib/install_db4.sh `pwd` CC=cc CXX=c++ ``` -### Resource limits - -The standard ulimit restrictions in OpenBSD are very strict: - - data(kbytes) 1572864 - -This is, unfortunately, no longer enough to compile some `.cpp` files in the project, -at least with gcc 4.9.3 (see issue https://github.com/bitcoin/bitcoin/issues/6658). -If your user is in the `staff` group the limit can be raised with: - - ulimit -d 3000000 +from the root of the repository. Then set `BDB_PREFIX` for the next section: -The change will only affect the current shell and processes spawned by it. To -make the change system-wide, change `datasize-cur` and `datasize-max` in -`/etc/login.conf`, and reboot. +```shell +export BDB_PREFIX="$PWD/db4" +``` ### Building Bitcoin Core @@ -118,59 +59,42 @@ export AUTOCONF_VERSION=2.69 # replace this with the autoconf version that you i export AUTOMAKE_VERSION=1.15 # replace this with the automake version that you installed ./autogen.sh ``` -Make sure `BDB_PREFIX` and `BOOST_PREFIX` are set to the appropriate paths from the above steps. +Make sure `BDB_PREFIX` is set to the appropriate path from the above steps. To configure with wallet: ```bash -./configure --with-gui=no --with-boost=$BOOST_PREFIX \ - CC=egcc CXX=eg++ CPP=ecpp \ +./configure --with-gui=no CC=cc CXX=c++ \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" ``` To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no --with-boost=$BOOST_PREFIX \ - CC=egcc CXX=eg++ CPP=ecpp +./configure --disable-wallet --with-gui=no CC=cc CXX=c++ ``` Build and run the tests: ```bash -gmake # can use -jX here for parallelism +gmake # use -jX here for parallelism gmake check ``` -Clang (not currently working) ------------------------------- - -WARNING: This is outdated, needs to be updated for OpenBSD 6.0 and re-tried. - -Using a newer g++ results in linking the new code to a new libstdc++. -Libraries built with the old g++, will still import the old library. -This gives conflicts, necessitating rebuild of all C++ dependencies of the application. +Resource limits +------------------- -With clang this can - at least theoretically - be avoided because it uses the -base system's libstdc++. - -```bash -pkg_add llvm boost -``` +If the build runs into out-of-memory errors, the instructions in this section +might help. -```bash -./configure --disable-wallet --with-gui=no CC=clang CXX=clang++ -gmake -``` +The standard ulimit restrictions in OpenBSD are very strict: -However, this does not appear to work. Compilation succeeds, but link fails -with many 'local symbol discarded' errors: + data(kbytes) 1572864 - local symbol 150: discarded in section `.text._ZN10tinyformat6detail14FormatIterator6finishEv' from libbitcoin_util.a(libbitcoin_util_a-random.o) - local symbol 151: discarded in section `.text._ZN10tinyformat6detail14FormatIterator21streamStateFromFormatERSoRjPKcii' from libbitcoin_util.a(libbitcoin_util_a-random.o) - local symbol 152: discarded in section `.text._ZN10tinyformat6detail12convertToIntIA13_cLb0EE6invokeERA13_Kc' from libbitcoin_util.a(libbitcoin_util_a-random.o) +This, unfortunately, in some cases not enough to compile some `.cpp` files in the project, +(see issue [#6658](https://github.com/bitcoin/bitcoin/issues/6658)). +If your user is in the `staff` group the limit can be raised with: -According to similar reported errors this is a binutils (ld) issue in 2.15, the -version installed by OpenBSD 5.7: + ulimit -d 3000000 -- http://openbsd-archive.7691.n7.nabble.com/UPDATE-cppcheck-1-65-td248900.html -- https://llvm.org/bugs/show_bug.cgi?id=9758 +The change will only affect the current shell and processes spawned by it. To +make the change system-wide, change `datasize-cur` and `datasize-max` in +`/etc/login.conf`, and reboot. -There is no known workaround for this. diff --git a/doc/build-osx.md b/doc/build-osx.md index 836c856a64..3e243933c8 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -16,7 +16,7 @@ Then install [Homebrew](https://brew.sh). Dependencies ---------------------- - brew install automake berkeley-db4 libtool boost --c++11 miniupnpc openssl pkg-config protobuf python3 qt libevent + brew install automake berkeley-db4 libtool boost miniupnpc openssl pkg-config protobuf python3 qt libevent See [dependencies.md](dependencies.md) for a complete overview. @@ -26,6 +26,20 @@ If you want to build the disk image with `make deploy` (.dmg / optional), you ne NOTE: Building with Qt4 is still supported, however, could result in a broken UI. Building with Qt5 is recommended. +Berkeley DB +----------- +It is recommended to use Berkeley DB 4.8. If you have to build it yourself, +you can use [the installation script included in contrib/](/contrib/install_db4.sh) +like so + +```shell +./contrib/install_db4.sh . +``` + +from the root of the repository. + +**Note**: You only need Berkeley DB if the wallet is enabled (see the section *Disable-Wallet mode* below). + Build Bitcoin Core ------------------------ @@ -96,6 +110,6 @@ Uncheck everything except Qt Creator during the installation process. Notes ----- -* Tested on OS X 10.8 through 10.12 on 64-bit Intel processors only. +* Tested on OS X 10.8 through 10.13 on 64-bit Intel processors only. * Building with downloaded Qt binaries is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714) diff --git a/doc/build-unix.md b/doc/build-unix.md index 8a102abaea..af567cadeb 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -165,33 +165,16 @@ turned off by default. See the configure options for upnp behavior desired: Berkeley DB ----------- -It is recommended to use Berkeley DB 4.8. If you have to build it yourself: +It is recommended to use Berkeley DB 4.8. If you have to build it yourself, +you can use [the installation script included in contrib/](/contrib/install_db4.sh) +like so -```bash -BITCOIN_ROOT=$(pwd) - -# Pick some path to install BDB to, here we create a directory within the bitcoin directory -BDB_PREFIX="${BITCOIN_ROOT}/db4" -mkdir -p $BDB_PREFIX - -# Fetch the source and verify that it is not tampered with -wget 'http://download.oracle.com/berkeley-db/db-4.8.30.NC.tar.gz' -echo '12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef db-4.8.30.NC.tar.gz' | sha256sum -c -# -> db-4.8.30.NC.tar.gz: OK -tar -xzvf db-4.8.30.NC.tar.gz - -# Build the library and install to our prefix -cd db-4.8.30.NC/build_unix/ -# Note: Do a static build so that it can be embedded into the executable, instead of having to find a .so at runtime -../dist/configure --enable-cxx --disable-shared --with-pic --prefix=$BDB_PREFIX -make install - -# Configure Bitcoin Core to use our own-built instance of BDB -cd $BITCOIN_ROOT -./autogen.sh -./configure LDFLAGS="-L${BDB_PREFIX}/lib/" CPPFLAGS="-I${BDB_PREFIX}/include/" # (other args...) +```shell +./contrib/install_db4.sh `pwd` ``` +from the root of the repository. + **Note**: You only need Berkeley DB if the wallet is enabled (see the section *Disable-Wallet mode* below). Boost @@ -329,17 +312,13 @@ You need to use GNU make (`gmake`) instead of `make`. For the wallet (optional): - pkg install db5 - -This will give a warning "configure: WARNING: Found Berkeley DB other -than 4.8; wallets opened by this build will not be portable!", but as FreeBSD never -had a binary release, this may not matter. If backwards compatibility -with 4.8-built Bitcoin Core is needed follow the steps under "Berkeley DB" above. + ./contrib/install_db4.sh `pwd` + setenv BDB_PREFIX $PWD/db4 Then build using: ./autogen.sh - ./configure --with-incompatible-bdb BDB_CFLAGS="-I/usr/local/include/db5" BDB_LIBS="-L/usr/local/lib -ldb_cxx-5" + ./configure BDB_CFLAGS="-I${BDB_PREFIX}/include" BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx" gmake *Note on debugging*: The version of `gdb` installed by default is [ancient and considered harmful](https://wiki.freebsd.org/GdbRetirement). diff --git a/doc/build-windows.md b/doc/build-windows.md index 0d96af26a2..9e0e66e522 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -3,61 +3,67 @@ WINDOWS BUILD NOTES Below are some notes on how to build Bitcoin Core for Windows. -Most developers use cross-compilation from Ubuntu to build executables for -Windows. Cross-compilation is also used to build the release binaries. +The options known to work for building Bitcoin Core on Windows are: -Currently only building on Ubuntu Trusty 14.04 or Ubuntu Zesty 17.04 or later is supported. -Building on Ubuntu Xenial 16.04 is known to be broken, see extensive discussion in issue [8732](https://github.com/bitcoin/bitcoin/issues/8732). -While it may be possible to do so with work arounds, it's potentially dangerous and not recommended. +* On Linux using the [Mingw-w64](https://mingw-w64.org/doku.php) cross compiler tool chain. Ubuntu Trusty 14.04 is recommended +and is the platform used to build the Bitcoin Core Windows release binaries. +* On Windows using [Windows +Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about) and the Mingw-w64 cross compiler tool chain. -While there are potentially a number of ways to build on Windows (for example using msys / mingw-w64), -using the Windows Subsystem For Linux is the most straightforward. If you are building with -another method, please contribute the instructions here for others who are running versions -of Windows that are not compatible with the Windows Subsystem for Linux. +Other options which may work but which have not been extensively tested are (please contribute instructions): -Compiling with Windows Subsystem For Linux -------------------------------------------- +* On Windows using a POSIX compatibility layer application such as [cygwin](http://www.cygwin.com/) or [msys2](http://www.msys2.org/). +* On Windows using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). + +Installing Windows Subsystem for Linux +--------------------------------------- With Windows 10, Microsoft has released a new feature named the [Windows -Subsystem for Linux](https://msdn.microsoft.com/commandline/wsl/about). This +Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about). This feature allows you to run a bash shell directly on Windows in an Ubuntu-based environment. Within this environment you can cross compile for Windows without -the need for a separate Linux VM or server. +the need for a separate Linux VM or server. Note that while WSL can be installed with +other Linux variants, such as OpenSUSE, the following instructions have only been +tested with Ubuntu. This feature is not supported in versions of Windows prior to Windows 10 or on Windows Server SKUs. In addition, it is available [only for 64-bit versions of Windows](https://msdn.microsoft.com/en-us/commandline/wsl/install_guide). -To get the bash shell, you must first activate the feature in Windows. +Full instructions to install WSL are available on the above link. +To install WSL on Windows 10 with Fall Creators Update installed (version >= 16215.0) do the following: -1. Turn on Developer Mode - * Open Settings -> Update and Security -> For developers - * Select the Developer Mode radio button - * Restart if necessary -2. Enable the Windows Subsystem for Linux feature +1. Enable the Windows Subsystem for Linux feature * From Start, search for "Turn Windows features on or off" (type 'turn') - * Select Windows Subsystem for Linux (beta) + * Select Windows Subsystem for Linux * Click OK * Restart if necessary +2. Install Ubuntu + * Open Microsoft Store and search for Ubuntu or use [this link](https://www.microsoft.com/store/productId/9NBLGGH4MSV6) + * Click Install 3. Complete Installation - * Open a cmd prompt and type "bash" - * Accept the license + * Open a cmd prompt and type "Ubuntu" * Create a new UNIX user account (this is a separate account from your Windows account) After the bash shell is active, you can follow the instructions below, starting with the "Cross-compilation" section. Compiling the 64-bit version is recommended but it is possible to compile the 32-bit version. -Cross-compilation -------------------- +Cross-compilation for Ubuntu and Windows Subsystem for Linux +------------------------------------------------------------ + +At the time of writing the Windows Subsystem for Linux installs Ubuntu Xenial 16.04. The Mingw-w64 package +for Ubuntu Xenial does not produce working executables for some of the Bitcoin Core applications. +It is possible to build on Ubuntu Xenial by installing the cross compiler packages from Ubuntu Zesty, see the steps below. +Building on Ubuntu Zesty 17.04 up to 17.10 has been verified to work. -These steps can be performed on, for example, an Ubuntu VM. The depends system +The steps below can be performed on Ubuntu (including in a VM) or WSL. The depends system will also work on other Linux distributions, however the commands for installing the toolchain will be different. First, install the general dependencies: - sudo apt-get install build-essential libtool autotools-dev automake pkg-config bsdmainutils curl + sudo apt install build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git A host toolchain (`build-essential`) is necessary because some dependency packages (such as `protobuf`) need to build host utilities that are used in the @@ -65,20 +71,44 @@ build process. See also: [dependencies.md](dependencies.md). -If you're building on Ubuntu 17.04 or later, run these two commands, selecting the 'posix' variant for both, -to work around issues with mingw-w64. See issue [8732](https://github.com/bitcoin/bitcoin/issues/8732) for more information. -``` -sudo update-alternatives --config x86_64-w64-mingw32-g++ -sudo update-alternatives --config x86_64-w64-mingw32-gcc -``` - ## Building for 64-bit Windows -To build executables for Windows 64-bit, install the following dependencies: +The first step is to install the mingw-w64 cross-compilation tool chain. Due to different Ubuntu +packages for each distribution and problems with the Xenial packages the steps for each are different. - sudo apt-get install g++-mingw-w64-x86-64 mingw-w64-x86-64-dev +Common steps to install mingw32 cross compiler tool chain: -Then build using: + sudo apt install g++-mingw-w64-x86-64 + +Ubuntu Trusty 14.04: + + No further steps required + +Ubuntu Xenial 16.04 and Windows Subsystem for Linux <sup>[1](#footnote1),[2](#footnote2)</sup>: + + sudo apt install software-properties-common + sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu zesty universe" + sudo apt update + sudo apt upgrade + sudo update-alternatives --config x86_64-w64-mingw32-g++ # Set the default mingw32 g++ compiler option to posix. + +Ubuntu Zesty 17.04 <sup>[2](#footnote2)</sup>: + + sudo update-alternatives --config x86_64-w64-mingw32-g++ # Set the default mingw32 g++ compiler option to posix. + +Once the tool chain is installed the build steps are common: + +Note that for WSL the Bitcoin Core source path MUST be somewhere in the default mount file system, for +example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. +This means you cannot use a directory that located directly on the host Windows file system to perform the build. + +The next three steps are an example of how to acquire the source in an appropriate way. + + cd /usr/src + sudo git clone https://github.com/bitcoin/bitcoin.git + sudo chmod -R a+rw bitcoin + +Once the source code is ready the build steps are below. PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var cd depends @@ -92,7 +122,21 @@ Then build using: To build executables for Windows 32-bit, install the following dependencies: - sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev + sudo apt install g++-mingw-w64-i686 mingw-w64-i686-dev + +For Ubuntu Xenial 16.04, Ubuntu Zesty 17.04 and Windows Subsystem for Linux <sup>[2](#footnote2)</sup>: + + sudo update-alternatives --config i686-w64-mingw32-g++ # Set the default mingw32 g++ compiler option to posix. + +Note that for WSL the Bitcoin Core source path MUST be somewhere in the default mount file system, for +example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. +This means you cannot use a directory that located directly on the host Windows file system to perform the build. + +The next three steps are an example of how to acquire the source in an appropriate way. + + cd /usr/src + sudo git clone https://github.com/bitcoin/bitcoin.git + sudo chmod -R a+rw bitcoin Then build using: @@ -117,3 +161,20 @@ as they appear in the release `.zip` archive. This can be done in the following way. This will install to `c:\workspace\bitcoin`, for example: make install DESTDIR=/mnt/c/workspace/bitcoin + +Footnotes +--------- + +<a name="footnote1">1</a>: There is currently a bug in the 64 bit Mingw-w64 cross compiler packaged for WSL/Ubuntu Xenial 16.04 that +causes two of the bitcoin executables to crash shortly after start up. The bug is related to the +-fstack-protector-all g++ compiler flag which is used to mitigate buffer overflows. +Installing the Mingw-w64 packages from the Ubuntu 17 distribution solves the issue, however, this is not +an officially supported approach and it's only recommended if you are prepared to reinstall WSL/Ubuntu should +something break. + +<a name="footnote2">2</a>: Starting from Ubuntu Xenial 16.04 both the 32 and 64 bit Mingw-w64 packages install two different +compiler options to allow a choice between either posix or win32 threads. The default option is win32 threads which is the more +efficient since it will result in binary code that links directly with the Windows kernel32.lib. Unfortunately, the headers +required to support win32 threads conflict with some of the classes in the C++11 standard library in particular std::mutex. +It's not possible to build the bitcoin code using the win32 version of the Mingw-w64 cross compilers (at least not without +modifying headers in the bitcoin source code). diff --git a/doc/dependencies.md b/doc/dependencies.md index 964c03ba88..5c5645de97 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -13,7 +13,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Expat | [2.2.1](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | | FreeType | [2.7.1](http://download.savannah.gnu.org/releases/freetype) | | No | | | -| GCC | | [4.7+](https://gcc.gnu.org/) | | | | +| GCC | | [4.8+](https://gcc.gnu.org/) | | | | | HarfBuzz-NG | | | | | | | libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libjpeg | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L75) | @@ -27,5 +27,5 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Qt | [5.7.1](https://download.qt.io/official_releases/qt/) | 4.7+ | No | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L94) (Linux only) | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L93) (Linux only) | -| ZeroMQ | [4.1.5](https://github.com/zeromq/libzmq/releases) | | No | | | +| ZeroMQ | [4.2.2](https://github.com/zeromq/libzmq/releases) | | No | | | | zlib | [1.2.11](http://zlib.net/) | | | | No | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 33c6ab9cb3..9dc63a1e4b 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -167,6 +167,37 @@ can be very difficult to track down. Compiling with -DDEBUG_LOCKORDER (configure CXXFLAGS="-DDEBUG_LOCKORDER -g") inserts run-time checks to keep track of which locks are held, and adds warnings to the debug.log file if inconsistencies are detected. +**Valgrind suppressions file** + +Valgrind is a programming tool for memory debugging, memory leak detection, and +profiling. The repo contains a Valgrind suppressions file +([`valgrind.supp`](https://github.com/bitcoin/bitcoin/blob/master/contrib/valgrind.supp)) +which includes known Valgrind warnings in our dependencies that cannot be fixed +in-tree. Example use: + +```shell +$ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin +$ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ + --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite +$ valgrind -v --leak-check=full src/bitcoind -printtoconsole +``` + +**compiling for test coverage** + +LCOV can be used to generate a test coverage report based upon `make check` +execution. LCOV must be installed on your system (e.g. the `lcov` package +on Debian/Ubuntu). + +To enable LCOV report generation during test runs: + +```shell +./configure --enable-lcov +make +make cov + +# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`. +``` + Locking/mutex usage notes ------------------------- @@ -461,6 +492,14 @@ namespace { - *Rationale*: Avoids confusion about the namespace context +- Prefer `#include <primitives/transaction.h>` bracket syntax instead of + `#include "primitives/transactions.h"` quote syntax when possible. + + - *Rationale*: Bracket syntax is less ambiguous because the preprocessor + searches a fixed list of include directories without taking location of the + source file into account. This allows quoted includes to stand out more when + the location of the source file actually is relevant. + GUI ----- @@ -644,3 +683,16 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: If a RPC response is not a JSON object then it is harder to avoid API breakage if new data in the response is needed. + +- Wallet RPCs call BlockUntilSyncedToCurrentChain to maintain consistency with + `getblockchaininfo`'s state immediately prior to the call's execution. Wallet + RPCs whose behavior does *not* depend on the current chainstate may omit this + call. + + - *Rationale*: In previous versions of Bitcoin Core, the wallet was always + in-sync with the chainstate (by virtue of them all being updated in the + same cs_main lock). In order to maintain the behavior that wallet RPCs + return results as of at least the highest best-known block an RPC + client may be aware of prior to entering a wallet RPC call, we must block + until the wallet is caught up to the chainstate as of the RPC call's entry. + This also makes the API much easier for RPC clients to reason about. diff --git a/doc/files.md b/doc/files.md index 3d603445fb..2eac7ed664 100644 --- a/doc/files.md +++ b/doc/files.md @@ -6,13 +6,16 @@ * blocks/rev000??.dat; block undo data (custom); since 0.8.0 (format changed since pre-0.8) * blocks/index/*; block index (LevelDB); since 0.8.0 * chainstate/*; block chain state database (LevelDB); since 0.8.0 -* database/*: BDB database environment; only used for wallet since 0.8.0 -* db.log: wallet database log file +* database/*: BDB database environment; only used for wallet since 0.8.0; moved to wallets/ directory on new installs since 0.16.0 +* db.log: wallet database log file; moved to wallets/ directory on new installs since 0.16.0 * debug.log: contains debug information and general logging generated by bitcoind or bitcoin-qt * fee_estimates.dat: stores statistics used to estimate minimum transaction fees and priorities required for confirmation; since 0.10.0 * mempool.dat: dump of the mempool's transactions; since 0.14.0. * peers.dat: peer IP address database (custom format); since 0.7.0 -* wallet.dat: personal wallet (BDB) with keys and transactions +* wallet.dat: personal wallet (BDB) with keys and transactions; moved to wallets/ directory on new installs since 0.16.0 +* wallets/database/*: BDB database environment; used for wallets since 0.16.0 +* wallets/db.log: wallet database log file; since 0.16.0 +* wallets/wallet.dat: personal wallet (BDB) with keys and transactions; since 0.16.0 * .cookie: session RPC authentication cookie (written at start when cookie authentication is used, deleted on shutdown): since 0.12.0 * onion_private_key: cached Tor hidden service private key for `-listenonion`: since 0.12.0 * guisettings.ini.bak: backup of former GUI settings after `-resetguisettings` is used diff --git a/doc/release-notes.md b/doc/release-notes.md index 4ecca7897c..d92666da72 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -20,7 +20,7 @@ How to Upgrade ============== If you are running an older version, shut it down. Wait until it has completely -shut down (which might take a few minutes for older versions), then run the +shut down (which might take a few minutes for older versions), then run the installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) or `bitcoind`/`bitcoin-qt` (on Linux). @@ -56,17 +56,10 @@ frequently tested on them. Notable changes =============== -Miner block size limiting deprecated ------------------------------------- - -Though blockmaxweight has been preferred for limiting the size of blocks returned by -getblocktemplate since 0.13.0, blockmaxsize remained as an option for those who wished -to limit their block size directly. Using this option resulted in a few UI issues as -well as non-optimal fee selection and ever-so-slightly worse performance, and has thus -now been deprecated. Further, the blockmaxsize option is now used only to calculate an -implied blockmaxweight, instead of limiting block size directly. Any miners who wish -to limit their blocks by size, instead of by weight, will have to do so manually by -removing transactions from their block template directly. +GCC 4.8.x +-------------- +The minimum version of GCC required to compile Bitcoin Core is now 4.8. No effort will be +made to support older versions of GCC. See discussion in issue #11732 for more information. HD-wallets by default --------------------- @@ -74,17 +67,50 @@ Due to a backward-incompatible change in the wallet database, wallets created with version 0.16.0 will be rejected by previous versions. Also, version 0.16.0 will only create hierarchical deterministic (HD) wallets. +Replace-By-Fee by default in GUI +-------------------------------- +The send screen now uses BIP-125 RBF by default, regardless of `-walletrbf`. +There is a checkbox to mark the transaction as final. + +The RPC default remains unchanged: to use RBF, launch with `-walletrbf=1` or +use the `replaceable` argument for individual transactions. + +Custom wallet directories +--------------------- +The ability to specify a directory other than the default data directory in which to store +wallets has been added. An existing directory can be specified using the `-walletdir=<dir>` +argument. Wallets loaded via `-wallet` arguments must be in this wallet directory. Care should be taken +when choosing a wallet directory location, as if it becomes unavailable during operation, +funds may be lost. + +Default wallet directory change +-------------------------- +On new installations (if the data directory doesn't exist), wallets will now be stored in a +new `wallets/` subdirectory inside the data directory. If this `wallets/` subdirectory +doesn't exist (i.e. on existing nodes), the current datadir root is used instead, as it was. + Low-level RPC changes ---------------------- -- The "currentblocksize" value in getmininginfo has been removed. - The deprecated RPC `getinfo` was removed. It is recommended that the more specific RPCs are used: * `getblockchaininfo` * `getnetworkinfo` * `getwalletinfo` * `getmininginfo` +- The wallet RPC `getreceivedbyaddress` will return an error if called with an address not in the wallet. + +Changed command-line options +----------------------------- +- `-debuglogfile=<file>` can be used to specify an alternative debug logging file. + +Renamed script for creating JSON-RPC credentials +----------------------------- +The `share/rpcuser/rpcuser.py` script was renamed to `share/rpcauth/rpcauth.py`. This script can be +used to create `rpcauth` credentials for a JSON-RPC user. + -- `dumpwallet` no longer allows overwriting files. This is a security measure - as well as prevents dangerous user mistakes. +- `dumpwallet` now includes hex-encoded scripts from the wallet in the dumpfile, and + `importwallet` now imports these scripts, but corresponding addresses may not be added + correctly or a manual rescan may be required to find relevant transactions. Credits ======= diff --git a/doc/release-notes/release-notes-0.15.0.md b/doc/release-notes/release-notes-0.15.0.md index 29816cacf7..3d000e94d5 100644 --- a/doc/release-notes/release-notes-0.15.0.md +++ b/doc/release-notes/release-notes-0.15.0.md @@ -151,8 +151,8 @@ Multi-wallet is enabled by using more than one `-wallet` argument when starting Bitcoin Core 0.15.0 contains the following changes to the RPC interface and `bitcoin-cli` for multi-wallet: * When running Bitcoin Core with a single wallet, there are **no** changes to the RPC interface or `bitcoin-cli`. All RPC calls and `bitcoin-cli` commands continue to work as before. -* When running Bitcoin Core with multi-wallet, all *node-level* RPC methods continue to work as before. HTTP RPC requests should be send to the normal `<RPC IP address>:<RPC port>/` endpoint, and `bitcoin-cli` commands should be run as before. A *node-level* RPC method is any method which does not require access to the wallet. -* When running Bitcoin Core with multi-wallet, *wallet-level* RPC methods must specify the wallet for which they're intended in every request. HTTP RPC requests should be send to the `<RPC IP address>:<RPC port>/wallet/<wallet name>/` endpoint, for example `127.0.0.1:8332/wallet/wallet1.dat/`. `bitcoin-cli` commands should be run with a `-rpcwallet` option, for example `bitcoin-cli -rpcwallet=wallet1.dat getbalance`. +* When running Bitcoin Core with multi-wallet, all *node-level* RPC methods continue to work as before. HTTP RPC requests should be send to the normal `<RPC IP address>:<RPC port>` endpoint, and `bitcoin-cli` commands should be run as before. A *node-level* RPC method is any method which does not require access to the wallet. +* When running Bitcoin Core with multi-wallet, *wallet-level* RPC methods must specify the wallet for which they're intended in every request. HTTP RPC requests should be send to the `<RPC IP address>:<RPC port>/wallet/<wallet name>` endpoint, for example `127.0.0.1:8332/wallet/wallet1.dat`. `bitcoin-cli` commands should be run with a `-rpcwallet` option, for example `bitcoin-cli -rpcwallet=wallet1.dat getbalance`. * A new *node-level* `listwallets` RPC method is added to display which wallets are currently loaded. The names returned by this method are the same as those used in the HTTP endpoint and for the `rpcwallet` argument. Note that while multi-wallet is now fully supported, the RPC multi-wallet interface should be considered unstable for version 0.15.0, and there may backwards-incompatible changes in future versions. diff --git a/doc/release-notes/release-notes-0.15.1.md b/doc/release-notes/release-notes-0.15.1.md new file mode 100644 index 0000000000..75d2e09714 --- /dev/null +++ b/doc/release-notes/release-notes-0.15.1.md @@ -0,0 +1,277 @@ +Bitcoin Core version *0.15.1* is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.15.1/> + +or + + <https://bitcoin.org/bin/bitcoin-core-0.15.1/> + +This is a new minor version release, including various bugfixes and +performance improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes for older versions), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +The first time you run version 0.15.0 or higher, your chainstate database will +be converted to a new format, which will take anywhere from a few minutes to +half an hour, depending on the speed of your machine. + +The file format of `fee_estimates.dat` changed in version 0.15.0. Hence, a +downgrade from version 0.15 or upgrade to version 0.15 will cause all fee +estimates to be discarded. + +Note that the block database format also changed in version 0.8.0 and there is no +automatic upgrade code from before version 0.8 to version 0.15.0. Upgrading +directly from 0.7.x and earlier without redownloading the blockchain is not supported. +However, as usual, old wallet versions are still supported. + +Downgrading warning +------------------- + +The chainstate database for this release is not compatible with previous +releases, so if you run 0.15 and then decide to switch back to any +older version, you will need to run the old release with the `-reindex-chainstate` +option to rebuild the chainstate data structures in the old format. + +If your node has pruning enabled, this will entail re-downloading and +processing the entire blockchain. + +Compatibility +============== + +Bitcoin Core is extensively tested on multiple operating systems using +the Linux kernel, macOS 10.8+, and Windows Vista and later. Windows XP is not supported. + +Bitcoin Core should also work on most other Unix-like systems but is not +frequently tested on them. + + +Notable changes +=============== + +Network fork safety enhancements +-------------------------------- + +A number of changes to the way Bitcoin Core deals with peer connections and invalid blocks +have been made, as a safety precaution against blockchain forks and misbehaving peers. + +- Unrequested blocks with less work than the minimum-chain-work are now no longer processed even +if they have more work than the tip (a potential issue during IBD where the tip may have low-work). +This prevents peers wasting the resources of a node. + +- Peers which provide a chain with less work than the minimum-chain-work during IBD will now be disconnected. + +- For a given outbound peer, we now check whether their best known block has at least as much work as our tip. If it +doesn't, and if we still haven't heard about a block with sufficient work after a 20 minute timeout, then we send +a single getheaders message, and wait 2 more minutes. If after two minutes their best known block has insufficient +work, we disconnect that peer. We protect 4 of our outbound peers from being disconnected by this logic to prevent +excessive network topology changes as a result of this algorithm, while still ensuring that we have a reasonable +number of nodes not known to be on bogus chains. + +- Outbound (non-manual) peers that serve us block headers that are already known to be invalid (other than compact +block announcements, because BIP 152 explicitly permits nodes to relay compact blocks before fully validating them) +will now be disconnected. + +- If the chain tip has not been advanced for over 30 minutes, we now assume the tip may be stale and will try to connect +to an additional outbound peer. A periodic check ensures that if this extra peer connection is in use, we will disconnect +the peer that least recently announced a new block. + +- The set of all known invalid-themselves blocks (i.e. blocks which we attempted to connect but which were found to be +invalid) are now tracked and used to check if new headers build on an invalid chain. This ensures that everything that +descends from an invalid block is marked as such. + + +Miner block size limiting deprecated +------------------------------------ + +Though blockmaxweight has been preferred for limiting the size of blocks returned by +getblocktemplate since 0.13.0, blockmaxsize remained as an option for those who wished +to limit their block size directly. Using this option resulted in a few UI issues as +well as non-optimal fee selection and ever-so-slightly worse performance, and has thus +now been deprecated. Further, the blockmaxsize option is now used only to calculate an +implied blockmaxweight, instead of limiting block size directly. Any miners who wish +to limit their blocks by size, instead of by weight, will have to do so manually by +removing transactions from their block template directly. + + +GUI settings backed up on reset +------------------------------- + +The GUI settings will now be written to `guisettings.ini.bak` in the data directory before wiping them when +the `-resetguisettings` argument is used. This can be used to retroactively troubleshoot issues due to the +GUI settings. + + +Duplicate wallets disallowed +---------------------------- + +Previously, it was possible to open the same wallet twice by manually copying the wallet file, causing +issues when both were opened simultaneously. It is no longer possible to open copies of the same wallet. + + +Debug `-minimumchainwork` argument added +---------------------------------------- + +A hidden debug argument `-minimumchainwork` has been added to allow a custom minimum work value to be used +when validating a chain. + + +Low-level RPC changes +---------------------- + +- The "currentblocksize" value in getmininginfo has been removed. + +- `dumpwallet` no longer allows overwriting files. This is a security measure + as well as prevents dangerous user mistakes. + +- `backupwallet` will now fail when attempting to backup to source file, rather than + destroying the wallet. + +- `listsinceblock` will now throw an error if an unknown `blockhash` argument + value is passed, instead of returning a list of all wallet transactions since + the genesis block. The behaviour is unchanged when an empty string is provided. + +0.15.1 Change log +================= + +### Mining +- #11100 `7871a7d` Fix confusing blockmax{size,weight} options, dont default to throwing away money (TheBlueMatt) + +### RPC and other APIs +- #10859 `2a5d099` gettxout: Slightly improve doc and tests (jtimon) +- #11267 `b1a6c94` update cli for estimate\*fee argument rename (laanwj) +- #11483 `20cdc2b` Fix importmulti bug when importing an already imported key (pedrobranco) +- #9937 `a43be5b` Prevent `dumpwallet` from overwriting files (laanwj) +- #11465 `405e069` Update named args documentation for importprivkey (dusty-wil) +- #11131 `b278a43` Write authcookie atomically (laanwj) +- #11565 `7d4546f` Make listsinceblock refuse unknown block hash (ryanofsky) +- #11593 `8195cb0` Work-around an upstream libevent bug (theuni) + +### P2P protocol and network code +- #11397 `27e861a` Improve and document SOCKS code (laanwj) +- #11252 `0fe2a9a` When clearing addrman clear mapInfo and mapAddr (instagibbs) +- #11527 `a2bd86a` Remove my testnet DNS seed (schildbach) +- #10756 `0a5477c` net processing: swap out signals for an interface class (theuni) +- #11531 `55b7abf` Check that new headers are not a descendant of an invalid block (more effeciently) (TheBlueMatt) +- #11560 `49bf090` Connect to a new outbound peer if our tip is stale (sdaftuar) +- #11568 `fc966bb` Disconnect outbound peers on invalid chains (sdaftuar) +- #11578 `ec8dedf` Add missing lock in ProcessHeadersMessage(...) (practicalswift) +- #11456 `6f27965` Replace relevant services logic with a function suite (TheBlueMatt) +- #11490 `bf191a7` Disconnect from outbound peers with bad headers chains (sdaftuar) + +### Validation +- #10357 `da4908c` Allow setting nMinimumChainWork on command line (sdaftuar) +- #11458 `2df65ee` Don't process unrequested, low-work blocks (sdaftuar) + +### Build system +- #11440 `b6c0209` Fix validationinterface build on super old boost/clang (TheBlueMatt) +- #11530 `265bb21` Add share/rpcuser to dist. source code archive (MarcoFalke) + +### GUI +- #11334 `19d63e8` Remove custom fee radio group and remove nCustomFeeRadio setting (achow101) +- #11198 `7310f1f` Fix display of package name on 'open config file' tooltip (esotericnonsense) +- #11015 `6642558` Add delay before filtering transactions (lclc) +- #11338 `6a62c74` Backup former GUI settings on `-resetguisettings` (laanwj) +- #11335 `8d13b42` Replace save|restoreWindowGeometry with Qt functions (MeshCollider) +- #11237 `2e31b1d` Fixing division by zero in time remaining (MeshCollider) +- #11247 `47c02a8` Use IsMine to validate custom change address (MarcoFalke) + +### Wallet +- #11017 `9e8aae3` Close DB on error (kallewoof) +- #11225 `6b4d9f2` Update stored witness in AddToWallet (sdaftuar) +- #11126 `2cb720a` Acquire cs_main lock before cs_wallet during wallet initialization (ryanofsky) +- #11476 `9c8006d` Avoid opening copied wallet databases simultaneously (ryanofsky) +- #11492 `de7053f` Fix leak in CDB constructor (promag) +- #11376 `fd79ed6` Ensure backupwallet fails when attempting to backup to source file (tomasvdw) +- #11326 `d570aa4` Fix crash on shutdown with invalid wallet (MeshCollider) + +### Tests and QA +- #11399 `a825d4a` Fix bip68-sequence rpc test (jl2012) +- #11150 `847c75e` Add getmininginfo test (mess110) +- #11407 `806c78f` add functional test for mempoolreplacement command line arg (instagibbs) +- #11433 `e169349` Restore bitcoin-util-test py2 compatibility (MarcoFalke) +- #11308 `2e1ac70` zapwallettxes: Wait up to 3s for mempool reload (MarcoFalke) +- #10798 `716066d` test bitcoin-cli (jnewbery) +- #11443 `019c492` Allow "make cov" out-of-tree; Fix rpc mapping check (MarcoFalke) +- #11445 `51bad91` 0.15.1 Backports (MarcoFalke) +- #11319 `2f0b30a` Fix error introduced into p2p-segwit.py, and prevent future similar errors (sdaftuar) +- #10552 `e4605d9` Tests for zmqpubrawtx and zmqpubrawblock (achow101) +- #11067 `eeb24a3` TestNode: Add wait_until_stopped helper method (MarcoFalke) +- #11068 `5398f20` Move wait_until to util (MarcoFalke) +- #11125 `812c870` Add bitcoin-cli -stdin and -stdinrpcpass functional tests (promag) +- #11077 `1d80d1e` fix timeout issues from TestNode (jnewbery) +- #11078 `f1ced0d` Make p2p-leaktests.py more robust (jnewbery) +- #11210 `f3f7891` Stop test_bitcoin-qt touching ~/.bitcoin (MeshCollider) +- #11234 `f0b6795` Remove redundant testutil.cpp|h files (MeshCollider) +- #11215 `cef0319` fixups from set_test_params() (jnewbery) +- #11345 `f9cf7b5` Check connectivity before sending in assumevalid.py (jnewbery) +- #11091 `c276c1e` Increase initial RPC timeout to 60 seconds (laanwj) +- #10711 `fc2aa09` Introduce TestNode (jnewbery) +- #11230 `d8dd8e7` Fixup dbcrash interaction with add_nodes() (jnewbery) +- #11241 `4424176` Improve signmessages functional test (mess110) +- #11116 `2c4ff35` Unit tests for script/standard and IsMine functions (jimpo) +- #11422 `a36f332` Verify DBWrapper iterators are taking snapshots (TheBlueMatt) +- #11121 `bb5e7cb` TestNode tidyups (jnewbery) +- #11521 `ca0f3f7` travis: move back to the minimal image (theuni) +- #11538 `adbc9d1` Fix race condition failures in replace-by-fee.py, sendheaders.py (sdaftuar) +- #11472 `4108879` Make tmpdir option an absolute path, misc cleanup (MarcoFalke) +- #10853 `5b728c8` Fix RPC failure testing (again) (jnewbery) +- #11310 `b6468d3` Test listwallets RPC (mess110) + +### Miscellaneous +- #11377 `75997c3` Disallow uncompressed pubkeys in bitcoin-tx [multisig] output adds (TheBlueMatt) +- #11437 `dea3b87` [Docs] Update Windows build instructions for using WSL and Ubuntu 17.04 (fanquake) +- #11318 `8b61aee` Put back inadvertently removed copyright notices (gmaxwell) +- #11442 `cf18f42` [Docs] Update OpenBSD Build Instructions for OpenBSD 6.2 (fanquake) +- #10957 `50bd3f6` Avoid returning a BIP9Stats object with uninitialized values (practicalswift) +- #11539 `01223a0` [verify-commits] Allow revoked keys to expire (TheBlueMatt) + + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andreas Schildbach +- Andrew Chow +- Chris Moore +- Cory Fields +- Cristian Mircea Messel +- Daniel Edgecumbe +- Donal OConnor +- Dusty Williams +- fanquake +- Gregory Sanders +- Jim Posen +- John Newbery +- Johnson Lau +- João Barbosa +- Jorge Timón +- Karl-Johan Alm +- Lucas Betschart +- MarcoFalke +- Matt Corallo +- Paul Berg +- Pedro Branco +- Pieter Wuille +- practicalswift +- Russell Yanofsky +- Samuel Dobson +- Suhas Daftuar +- Tomas van der Wansem +- Wladimir J. van der Laan + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). diff --git a/share/genbuild.sh b/share/genbuild.sh index 32ef2a5755..419e0da0fd 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. if [ $# -gt 1 ]; then - cd "$2" + cd "$2" || exit 1 fi if [ $# -gt 0 ]; then FILE="$1" diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index e3be18205d..fbf3ea3436 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2012-2016 The Bitcoin Core developers +# Copyright (c) 2012-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' diff --git a/share/rpcauth/README.md b/share/rpcauth/README.md new file mode 100644 index 0000000000..389278a125 --- /dev/null +++ b/share/rpcauth/README.md @@ -0,0 +1,10 @@ +RPC Tools +--------------------- + +### [RPCAuth](/share/rpcauth) ### + +Create login credentials for a JSON-RPC user. + +Usage: + + ./rpcauth.py <username> diff --git a/share/rpcuser/rpcuser.py b/share/rpcauth/rpcauth.py index 63c69e308a..7baad85582 100755 --- a/share/rpcuser/rpcuser.py +++ b/share/rpcauth/rpcauth.py @@ -1,5 +1,5 @@ -#!/usr/bin/env python2 -# Copyright (c) 2015-2016 The Bitcoin Core developers +#!/usr/bin/env python +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/share/rpcuser/README.md b/share/rpcuser/README.md deleted file mode 100644 index 12a8e6fb0c..0000000000 --- a/share/rpcuser/README.md +++ /dev/null @@ -1,10 +0,0 @@ -RPC Tools ---------------------- - -### [RPCUser](/share/rpcuser) ### - -Create an RPC user login credential. - -Usage: - - ./rpcuser.py <username> diff --git a/src/Makefile.am b/src/Makefile.am index 90deff48b0..4b65774fc6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ else LIBUNIVALUE = $(UNIVALUE_LIBS) endif -BITCOIN_INCLUDES=-I$(builddir) -I$(builddir)/obj $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) $(CRYPTO_CFLAGS) $(SSL_CFLAGS) BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) @@ -168,6 +168,7 @@ BITCOIN_CORE_H = \ wallet/rpcwallet.h \ wallet/wallet.h \ wallet/walletdb.h \ + wallet/walletutil.h \ warnings.h \ zmq/zmqabstractnotifier.h \ zmq/zmqconfig.h\ @@ -249,6 +250,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/rpcwallet.cpp \ wallet/wallet.cpp \ wallet/walletdb.cpp \ + wallet/walletutil.cpp \ $(BITCOIN_CORE_H) # crypto primitives library @@ -500,10 +502,6 @@ clean-local: ## FIXME: How to get the appropriate modulename_CPPFLAGS in here? $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ -.mm.o: - $(AM_V_CXX) $(OBJCXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ - $(CPPFLAGS) $(AM_CXXFLAGS) $(QT_INCLUDES) $(AM_CXXFLAGS) $(PIE_FLAGS) $(CXXFLAGS) -c -o $@ $< - check-symbols: $(bin_PROGRAMS) if GLIBC_BACK_COMPAT @echo "Checking glibc back compat..." diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index e4b64c1ca7..0bdde06772 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -362,12 +362,12 @@ RES_MOVIES = $(wildcard $(srcdir)/qt/res/movies/spinner-*.png) BITCOIN_RC = qt/res/bitcoin-qt-res.rc -BITCOIN_QT_INCLUDES = -I$(builddir)/qt -I$(srcdir)/qt -I$(srcdir)/qt/forms \ - -I$(builddir)/qt/forms -DQT_NO_KEYWORDS +BITCOIN_QT_INCLUDES = -DQT_NO_KEYWORDS qt_libbitcoinqt_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(QT_DBUS_INCLUDES) $(PROTOBUF_CFLAGS) $(QR_CFLAGS) qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) +qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(PROTOBUF_PROTO) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) @@ -438,7 +438,7 @@ $(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_ICONS) $(RES_IMAGES) $(RES_MOVIES) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin $< | \ $(SED) -e '/^\*\*.*Created:/d' -e '/^\*\*.*by:/d' > $@ -CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno +CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno qt/temp_bitcoin_locale.qrc CLEANFILES += $(CLEAN_QT) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d3e7b5da12..d4d972b2bb 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -33,6 +33,7 @@ BITCOIN_TESTS =\ test/base64_tests.cpp \ test/bech32_tests.cpp \ test/bip32_tests.cpp \ + test/blockchain_tests.cpp \ test/blockencodings_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ @@ -80,10 +81,10 @@ BITCOIN_TESTS =\ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ + test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ test/versionbits_tests.cpp \ test/uint256_tests.cpp \ - test/univalue_tests.cpp \ test/util_tests.cpp if ENABLE_WALLET @@ -96,7 +97,7 @@ BITCOIN_TESTS += \ endif test_test_bitcoin_SOURCES = $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) -test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -I$(builddir)/test/ $(TESTDEFS) $(EVENT_CFLAGS) +test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(EVENT_CFLAGS) test_test_bitcoin_LDADD = if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 7f85c16585..675f3c28af 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -1,19 +1,18 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addrdb.h" - -#include "addrman.h" -#include "chainparams.h" -#include "clientversion.h" -#include "fs.h" -#include "hash.h" -#include "random.h" -#include "streams.h" -#include "tinyformat.h" -#include "util.h" +#include <addrdb.h> + +#include <addrman.h> +#include <chainparams.h> +#include <clientversion.h> +#include <hash.h> +#include <random.h> +#include <streams.h> +#include <tinyformat.h> +#include <util.h> namespace { diff --git a/src/addrdb.h b/src/addrdb.h index d930de204d..e39bd6f5ae 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_ADDRDB_H #define BITCOIN_ADDRDB_H -#include "fs.h" -#include "serialize.h" +#include <fs.h> +#include <serialize.h> #include <string> #include <map> diff --git a/src/addrman.cpp b/src/addrman.cpp index a56bb4f9c1..5a769ac93a 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,13 +1,13 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addrman.h" +#include <addrman.h> -#include "hash.h" -#include "serialize.h" -#include "streams.h" +#include <hash.h> +#include <serialize.h> +#include <streams.h> int CAddrInfo::GetTriedBucket(const uint256& nKey) const { @@ -390,9 +390,9 @@ int CAddrMan::Check_() if (vRandom.size() != nTried + nNew) return -7; - for (std::map<int, CAddrInfo>::iterator it = mapInfo.begin(); it != mapInfo.end(); it++) { - int n = (*it).first; - CAddrInfo& info = (*it).second; + for (const auto& entry : mapInfo) { + int n = entry.first; + const CAddrInfo& info = entry.second; if (info.fInTried) { if (!info.nLastSuccess) return -1; diff --git a/src/addrman.h b/src/addrman.h index 18f3062287..172cb5f45b 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -1,17 +1,17 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H -#include "netaddress.h" -#include "protocol.h" -#include "random.h" -#include "sync.h" -#include "timedata.h" -#include "util.h" +#include <netaddress.h> +#include <protocol.h> +#include <random.h> +#include <sync.h> +#include <timedata.h> +#include <util.h> #include <map> #include <set> @@ -313,9 +313,9 @@ public: s << nUBuckets; std::map<int, int> mapUnkIds; int nIds = 0; - for (std::map<int, CAddrInfo>::const_iterator it = mapInfo.begin(); it != mapInfo.end(); it++) { - mapUnkIds[(*it).first] = nIds; - const CAddrInfo &info = (*it).second; + for (const auto& entry : mapInfo) { + mapUnkIds[entry.first] = nIds; + const CAddrInfo &info = entry.second; if (info.nRefCount) { assert(nIds != nNew); // this means nNew was wrong, oh ow s << info; @@ -323,8 +323,8 @@ public: } } nIds = 0; - for (std::map<int, CAddrInfo>::const_iterator it = mapInfo.begin(); it != mapInfo.end(); it++) { - const CAddrInfo &info = (*it).second; + for (const auto& entry : mapInfo) { + const CAddrInfo &info = entry.second; if (info.fInTried) { assert(nIds != nTried); // this means nTried was wrong, oh ow s << info; @@ -455,6 +455,7 @@ public: void Clear() { + LOCK(cs); std::vector<int>().swap(vRandom); nKey = GetRandHash(); for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index b4952af6f4..65de632306 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "arith_uint256.h" +#include <arith_uint256.h> -#include "uint256.h" -#include "utilstrencodings.h" -#include "crypto/common.h" +#include <uint256.h> +#include <utilstrencodings.h> +#include <crypto/common.h> #include <stdio.h> #include <string.h> diff --git a/src/arith_uint256.h b/src/arith_uint256.h index 5fd4fe96cf..dc2627592e 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -25,7 +25,7 @@ template<unsigned int BITS> class base_uint { protected: - enum { WIDTH=BITS/32 }; + static constexpr int WIDTH = BITS / 32; uint32_t pn[WIDTH]; public: diff --git a/src/base58.cpp b/src/base58.cpp index 9d5a2f4964..499afbe382 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" +#include <base58.h> -#include "bech32.h" -#include "hash.h" -#include "script/script.h" -#include "uint256.h" -#include "utilstrencodings.h" +#include <bech32.h> +#include <hash.h> +#include <script/script.h> +#include <uint256.h> +#include <utilstrencodings.h> #include <boost/variant/apply_visitor.hpp> #include <boost/variant/static_visitor.hpp> diff --git a/src/base58.h b/src/base58.h index 9dc4234248..39eb4eacc2 100644 --- a/src/base58.h +++ b/src/base58.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,11 +14,11 @@ #ifndef BITCOIN_BASE58_H #define BITCOIN_BASE58_H -#include "chainparams.h" -#include "key.h" -#include "pubkey.h" -#include "script/standard.h" -#include "support/allocators/zeroafterfree.h" +#include <chainparams.h> +#include <key.h> +#include <pubkey.h> +#include <script/standard.h> +#include <support/allocators/zeroafterfree.h> #include <string> #include <vector> diff --git a/src/bech32.cpp b/src/bech32.cpp index 573eac58bb..274782e467 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bech32.h" +#include <bech32.h> namespace { diff --git a/src/bench/Examples.cpp b/src/bench/Examples.cpp index 314947d48c..b68c9cd156 100644 --- a/src/bench/Examples.cpp +++ b/src/bench/Examples.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "validation.h" -#include "utiltime.h" +#include <bench/bench.h> +#include <validation.h> +#include <utiltime.h> // Sanity test: this should loop ten times, and // min/max/average should be close to 100ms. @@ -15,7 +15,7 @@ static void Sleep100ms(benchmark::State& state) } } -BENCHMARK(Sleep100ms); +BENCHMARK(Sleep100ms, 10); // Extremely fast-running benchmark: #include <math.h> @@ -31,4 +31,4 @@ static void Trig(benchmark::State& state) } } -BENCHMARK(Trig); +BENCHMARK(Trig, 12 * 1000 * 1000); diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 65e27a615d..70bfd7d0bf 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" +#include <bench/bench.h> -#include "validation.h" -#include "base58.h" +#include <validation.h> +#include <base58.h> #include <array> #include <vector> @@ -22,7 +22,7 @@ static void Base58Encode(benchmark::State& state) } }; while (state.KeepRunning()) { - EncodeBase58(buff.begin(), buff.end()); + EncodeBase58(buff.data(), buff.data() + buff.size()); } } @@ -54,6 +54,6 @@ static void Base58Decode(benchmark::State& state) } -BENCHMARK(Base58Encode); -BENCHMARK(Base58CheckEncode); -BENCHMARK(Base58Decode); +BENCHMARK(Base58Encode, 470 * 1000); +BENCHMARK(Base58CheckEncode, 320 * 1000); +BENCHMARK(Base58Decode, 800 * 1000); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 7b307d6f42..dd120f3590 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -1,104 +1,143 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "perf.h" +#include <bench/bench.h> +#include <bench/perf.h> #include <assert.h> #include <iostream> #include <iomanip> -#include <sys/time.h> +#include <algorithm> +#include <regex> +#include <numeric> -benchmark::BenchRunner::BenchmarkMap &benchmark::BenchRunner::benchmarks() { - static std::map<std::string, benchmark::BenchFunction> benchmarks_map; - return benchmarks_map; +void benchmark::ConsolePrinter::header() +{ + std::cout << "# Benchmark, evals, iterations, total, min, max, median" << std::endl; } -static double gettimedouble(void) { - struct timeval tv; - gettimeofday(&tv, nullptr); - return tv.tv_usec * 0.000001 + tv.tv_sec; +void benchmark::ConsolePrinter::result(const State& state) +{ + auto results = state.m_elapsed_results; + std::sort(results.begin(), results.end()); + + double total = state.m_num_iters * std::accumulate(results.begin(), results.end(), 0.0); + + double front = 0; + double back = 0; + double median = 0; + + if (!results.empty()) { + front = results.front(); + back = results.back(); + + size_t mid = results.size() / 2; + median = results[mid]; + if (0 == results.size() % 2) { + median = (results[mid] + results[mid + 1]) / 2; + } + } + + std::cout << std::setprecision(6); + std::cout << state.m_name << ", " << state.m_num_evals << ", " << state.m_num_iters << ", " << total << ", " << front << ", " << back << ", " << median << std::endl; } -benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func) +void benchmark::ConsolePrinter::footer() {} +benchmark::PlotlyPrinter::PlotlyPrinter(std::string plotly_url, int64_t width, int64_t height) + : m_plotly_url(plotly_url), m_width(width), m_height(height) { - benchmarks().insert(std::make_pair(name, func)); } -void -benchmark::BenchRunner::RunAll(double elapsedTimeForOne) +void benchmark::PlotlyPrinter::header() { - perf_init(); - std::cout << "#Benchmark" << "," << "count" << "," << "min" << "," << "max" << "," << "average" << "," - << "min_cycles" << "," << "max_cycles" << "," << "average_cycles" << "\n"; + std::cout << "<html><head>" + << "<script src=\"" << m_plotly_url << "\"></script>" + << "</head><body><div id=\"myDiv\" style=\"width:" << m_width << "px; height:" << m_height << "px\"></div>" + << "<script> var data = [" + << std::endl; +} - for (const auto &p: benchmarks()) { - State state(p.first, elapsedTimeForOne); - p.second(state); +void benchmark::PlotlyPrinter::result(const State& state) +{ + std::cout << "{ " << std::endl + << " name: '" << state.m_name << "', " << std::endl + << " y: ["; + + const char* prefix = ""; + for (const auto& e : state.m_elapsed_results) { + std::cout << prefix << std::setprecision(6) << e; + prefix = ", "; } - perf_fini(); + std::cout << "]," << std::endl + << " boxpoints: 'all', jitter: 0.3, pointpos: 0, type: 'box'," + << std::endl + << "}," << std::endl; } -bool benchmark::State::KeepRunning() +void benchmark::PlotlyPrinter::footer() { - if (count & countMask) { - ++count; - return true; - } - double now; - uint64_t nowCycles; - if (count == 0) { - lastTime = beginTime = now = gettimedouble(); - lastCycles = beginCycles = nowCycles = perf_cpucycles(); + std::cout << "]; var layout = { showlegend: false, yaxis: { rangemode: 'tozero', autorange: true } };" + << "Plotly.newPlot('myDiv', data, layout);" + << "</script></body></html>"; +} + + +benchmark::BenchRunner::BenchmarkMap& benchmark::BenchRunner::benchmarks() +{ + static std::map<std::string, Bench> benchmarks_map; + return benchmarks_map; +} + +benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func, uint64_t num_iters_for_one_second) +{ + benchmarks().insert(std::make_pair(name, Bench{func, num_iters_for_one_second})); +} + +void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double scaling, const std::string& filter, bool is_list_only) +{ + perf_init(); + if (!std::ratio_less_equal<benchmark::clock::period, std::micro>::value) { + std::cerr << "WARNING: Clock precision is worse than microsecond - benchmarks may be less accurate!\n"; } - else { - now = gettimedouble(); - double elapsed = now - lastTime; - double elapsedOne = elapsed / (countMask + 1); - if (elapsedOne < minTime) minTime = elapsedOne; - if (elapsedOne > maxTime) maxTime = elapsedOne; - - // We only use relative values, so don't have to handle 64-bit wrap-around specially - nowCycles = perf_cpucycles(); - uint64_t elapsedOneCycles = (nowCycles - lastCycles) / (countMask + 1); - if (elapsedOneCycles < minCycles) minCycles = elapsedOneCycles; - if (elapsedOneCycles > maxCycles) maxCycles = elapsedOneCycles; - - if (elapsed*128 < maxElapsed) { - // If the execution was much too fast (1/128th of maxElapsed), increase the count mask by 8x and restart timing. - // The restart avoids including the overhead of this code in the measurement. - countMask = ((countMask<<3)|7) & ((1LL<<60)-1); - count = 0; - minTime = std::numeric_limits<double>::max(); - maxTime = std::numeric_limits<double>::min(); - minCycles = std::numeric_limits<uint64_t>::max(); - maxCycles = std::numeric_limits<uint64_t>::min(); - return true; + + std::regex reFilter(filter); + std::smatch baseMatch; + + printer.header(); + + for (const auto& p : benchmarks()) { + if (!std::regex_match(p.first, baseMatch, reFilter)) { + continue; } - if (elapsed*16 < maxElapsed) { - uint64_t newCountMask = ((countMask<<1)|1) & ((1LL<<60)-1); - if ((count & newCountMask)==0) { - countMask = newCountMask; - } + + uint64_t num_iters = static_cast<uint64_t>(p.second.num_iters_for_one_second * scaling); + if (0 == num_iters) { + num_iters = 1; } + State state(p.first, num_evals, num_iters, printer); + if (!is_list_only) { + p.second.func(state); + } + printer.result(state); } - lastTime = now; - lastCycles = nowCycles; - ++count; - if (now - beginTime < maxElapsed) return true; // Keep going + printer.footer(); - --count; + perf_fini(); +} - assert(count != 0 && "count == 0 => (now == 0 && beginTime == 0) => return above"); +bool benchmark::State::UpdateTimer(const benchmark::time_point current_time) +{ + if (m_start_time != time_point()) { + std::chrono::duration<double> diff = current_time - m_start_time; + m_elapsed_results.push_back(diff.count() / m_num_iters); - // Output results - double average = (now-beginTime)/count; - int64_t averageCycles = (nowCycles-beginCycles)/count; - std::cout << std::fixed << std::setprecision(15) << name << "," << count << "," << minTime << "," << maxTime << "," << average << "," - << minCycles << "," << maxCycles << "," << averageCycles << "\n"; - std::cout.copyfmt(std::ios(nullptr)); + if (m_elapsed_results.size() == m_num_evals) { + return false; + } + } - return false; + m_num_iters_left = m_num_iters - 1; + return true; } diff --git a/src/bench/bench.h b/src/bench/bench.h index 79109eaa56..e15cb81869 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,6 +9,8 @@ #include <limits> #include <map> #include <string> +#include <vector> +#include <chrono> #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> @@ -31,50 +33,110 @@ static void CODE_TO_TIME(benchmark::State& state) ... do any cleanup needed... } -BENCHMARK(CODE_TO_TIME); +// default to running benchmark for 5000 iterations +BENCHMARK(CODE_TO_TIME, 5000); */ - + namespace benchmark { +// In case high_resolution_clock is steady, prefer that, otherwise use steady_clock. +struct best_clock { + using hi_res_clock = std::chrono::high_resolution_clock; + using steady_clock = std::chrono::steady_clock; + using type = std::conditional<hi_res_clock::is_steady, hi_res_clock, steady_clock>::type; +}; +using clock = best_clock::type; +using time_point = clock::time_point; +using duration = clock::duration; - class State { - std::string name; - double maxElapsed; - double beginTime; - double lastTime, minTime, maxTime; - uint64_t count; - uint64_t countMask; - uint64_t beginCycles; - uint64_t lastCycles; - uint64_t minCycles; - uint64_t maxCycles; - public: - State(std::string _name, double _maxElapsed) : name(_name), maxElapsed(_maxElapsed), count(0) { - minTime = std::numeric_limits<double>::max(); - maxTime = std::numeric_limits<double>::min(); - minCycles = std::numeric_limits<uint64_t>::max(); - maxCycles = std::numeric_limits<uint64_t>::min(); - countMask = 1; - } - bool KeepRunning(); - }; +class Printer; - typedef std::function<void(State&)> BenchFunction; +class State +{ +public: + std::string m_name; + uint64_t m_num_iters_left; + const uint64_t m_num_iters; + const uint64_t m_num_evals; + std::vector<double> m_elapsed_results; + time_point m_start_time; + + bool UpdateTimer(time_point finish_time); - class BenchRunner + State(std::string name, uint64_t num_evals, double num_iters, Printer& printer) : m_name(name), m_num_iters_left(0), m_num_iters(num_iters), m_num_evals(num_evals) { - typedef std::map<std::string, BenchFunction> BenchmarkMap; - static BenchmarkMap &benchmarks(); + } + + inline bool KeepRunning() + { + if (m_num_iters_left--) { + return true; + } + + bool result = UpdateTimer(clock::now()); + // measure again so runtime of UpdateTimer is not included + m_start_time = clock::now(); + return result; + } +}; - public: - BenchRunner(std::string name, BenchFunction func); +typedef std::function<void(State&)> BenchFunction; - static void RunAll(double elapsedTimeForOne=1.0); +class BenchRunner +{ + struct Bench { + BenchFunction func; + uint64_t num_iters_for_one_second; }; + typedef std::map<std::string, Bench> BenchmarkMap; + static BenchmarkMap& benchmarks(); + +public: + BenchRunner(std::string name, BenchFunction func, uint64_t num_iters_for_one_second); + + static void RunAll(Printer& printer, uint64_t num_evals, double scaling, const std::string& filter, bool is_list_only); +}; + +// interface to output benchmark results. +class Printer +{ +public: + virtual ~Printer() {} + virtual void header() = 0; + virtual void result(const State& state) = 0; + virtual void footer() = 0; +}; + +// default printer to console, shows min, max, median. +class ConsolePrinter : public Printer +{ +public: + void header(); + void result(const State& state); + void footer(); +}; + +// creates box plot with plotly.js +class PlotlyPrinter : public Printer +{ +public: + PlotlyPrinter(std::string plotly_url, int64_t width, int64_t height); + void header(); + void result(const State& state); + void footer(); + +private: + std::string m_plotly_url; + int64_t m_width; + int64_t m_height; +}; } -// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); -#define BENCHMARK(n) \ - benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n); + +// BENCHMARK(foo, num_iters_for_one_second) expands to: benchmark::BenchRunner bench_11foo("foo", num_iterations); +// Choose a num_iters_for_one_second that takes roughly 1 second. The goal is that all benchmarks should take approximately +// the same time, and scaling factor can be used that the total time is appropriate for your system. +#define BENCHMARK(n, num_iters_for_one_second) \ + benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n, (num_iters_for_one_second)); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 37fd772435..fc92a46c0f 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -1,25 +1,71 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" +#include <bench/bench.h> -#include "crypto/sha256.h" -#include "key.h" -#include "validation.h" -#include "util.h" -#include "random.h" +#include <crypto/sha256.h> +#include <key.h> +#include <validation.h> +#include <util.h> +#include <random.h> + +#include <boost/lexical_cast.hpp> + +#include <memory> + +static const int64_t DEFAULT_BENCH_EVALUATIONS = 5; +static const char* DEFAULT_BENCH_FILTER = ".*"; +static const char* DEFAULT_BENCH_SCALING = "1.0"; +static const char* DEFAULT_BENCH_PRINTER = "console"; +static const char* DEFAULT_PLOT_PLOTLYURL = "https://cdn.plot.ly/plotly-latest.min.js"; +static const int64_t DEFAULT_PLOT_WIDTH = 1024; +static const int64_t DEFAULT_PLOT_HEIGHT = 768; int main(int argc, char** argv) { + gArgs.ParseParameters(argc, argv); + + if (gArgs.IsArgSet("-?") || gArgs.IsArgSet("-h") || gArgs.IsArgSet("-help")) { + std::cout << HelpMessageGroup(_("Options:")) + << HelpMessageOpt("-?", _("Print this help message and exit")) + << HelpMessageOpt("-list", _("List benchmarks without executing them. Can be combined with -scaling and -filter")) + << HelpMessageOpt("-evals=<n>", strprintf(_("Number of measurement evaluations to perform. (default: %u)"), DEFAULT_BENCH_EVALUATIONS)) + << HelpMessageOpt("-filter=<regex>", strprintf(_("Regular expression filter to select benchmark by name (default: %s)"), DEFAULT_BENCH_FILTER)) + << HelpMessageOpt("-scaling=<n>", strprintf(_("Scaling factor for benchmark's runtime (default: %u)"), DEFAULT_BENCH_SCALING)) + << HelpMessageOpt("-printer=(console|plot)", strprintf(_("Choose printer format. console: print data to console. plot: Print results as HTML graph (default: %s)"), DEFAULT_BENCH_PRINTER)) + << HelpMessageOpt("-plot-plotlyurl=<uri>", strprintf(_("URL to use for plotly.js (default: %s)"), DEFAULT_PLOT_PLOTLYURL)) + << HelpMessageOpt("-plot-width=<x>", strprintf(_("Plot width in pixel (default: %u)"), DEFAULT_PLOT_WIDTH)) + << HelpMessageOpt("-plot-height=<x>", strprintf(_("Plot height in pixel (default: %u)"), DEFAULT_PLOT_HEIGHT)); + + return 0; + } + SHA256AutoDetect(); RandomInit(); ECC_Start(); SetupEnvironment(); fPrintToDebugLog = false; // don't want to write to debug.log file - benchmark::BenchRunner::RunAll(); + int64_t evaluations = gArgs.GetArg("-evals", DEFAULT_BENCH_EVALUATIONS); + std::string regex_filter = gArgs.GetArg("-filter", DEFAULT_BENCH_FILTER); + std::string scaling_str = gArgs.GetArg("-scaling", DEFAULT_BENCH_SCALING); + bool is_list_only = gArgs.GetBoolArg("-list", false); + + double scaling_factor = boost::lexical_cast<double>(scaling_str); + + + std::unique_ptr<benchmark::Printer> printer(new benchmark::ConsolePrinter()); + std::string printer_arg = gArgs.GetArg("-printer", DEFAULT_BENCH_PRINTER); + if ("plot" == printer_arg) { + printer.reset(new benchmark::PlotlyPrinter( + gArgs.GetArg("-plot-plotlyurl", DEFAULT_PLOT_PLOTLYURL), + gArgs.GetArg("-plot-width", DEFAULT_PLOT_WIDTH), + gArgs.GetArg("-plot-height", DEFAULT_PLOT_HEIGHT))); + } + + benchmark::BenchRunner::RunAll(*printer, evaluations, scaling_factor, regex_filter, is_list_only); ECC_Stop(); } diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 5aab3381fd..74169bcad7 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "coins.h" -#include "policy/policy.h" -#include "wallet/crypter.h" +#include <bench/bench.h> +#include <coins.h> +#include <policy/policy.h> +#include <wallet/crypter.h> #include <vector> @@ -84,4 +84,4 @@ static void CCoinsCaching(benchmark::State& state) } } -BENCHMARK(CCoinsCaching); +BENCHMARK(CCoinsCaching, 170 * 1000); diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 7bb1b93668..fac7e079a7 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -1,16 +1,16 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" +#include <bench/bench.h> -#include "chainparams.h" -#include "validation.h" -#include "streams.h" -#include "consensus/validation.h" +#include <chainparams.h> +#include <validation.h> +#include <streams.h> +#include <consensus/validation.h> namespace block_bench { -#include "bench/data/block413567.raw.h" +#include <bench/data/block413567.raw.h> } // namespace block_bench // These are the two major time-sinks which happen after we have fully received @@ -52,5 +52,5 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) } } -BENCHMARK(DeserializeBlockTest); -BENCHMARK(DeserializeAndCheckBlockTest); +BENCHMARK(DeserializeBlockTest, 130); +BENCHMARK(DeserializeAndCheckBlockTest, 160); diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index b7ae5c2d57..6e816f1bec 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -1,62 +1,22 @@ -// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "util.h" -#include "validation.h" -#include "checkqueue.h" -#include "prevector.h" +#include <bench/bench.h> +#include <util.h> +#include <validation.h> +#include <checkqueue.h> +#include <prevector.h> #include <vector> #include <boost/thread/thread.hpp> -#include "random.h" +#include <random.h> -// This Benchmark tests the CheckQueue with the lightest -// weight Checks, so it should make any lock contention -// particularly visible static const int MIN_CORES = 2; static const size_t BATCHES = 101; static const size_t BATCH_SIZE = 30; static const int PREVECTOR_SIZE = 28; -static const int QUEUE_BATCH_SIZE = 128; -static void CCheckQueueSpeed(benchmark::State& state) -{ - struct FakeJobNoWork { - bool operator()() - { - return true; - } - void swap(FakeJobNoWork& x){}; - }; - CCheckQueue<FakeJobNoWork> queue {QUEUE_BATCH_SIZE}; - boost::thread_group tg; - for (auto x = 0; x < std::max(MIN_CORES, GetNumCores()); ++x) { - tg.create_thread([&]{queue.Thread();}); - } - while (state.KeepRunning()) { - CCheckQueueControl<FakeJobNoWork> control(&queue); - - // We call Add a number of times to simulate the behavior of adding - // a block of transactions at once. - - std::vector<std::vector<FakeJobNoWork>> vBatches(BATCHES); - for (auto& vChecks : vBatches) { - vChecks.resize(BATCH_SIZE); - } - for (auto& vChecks : vBatches) { - // We can't make vChecks in the inner loop because we want to measure - // the cost of getting the memory to each thread and we might get the same - // memory - control.Add(vChecks); - } - // control waits for completion by RAII, but - // it is done explicitly here for clarity - control.Wait(); - } - tg.interrupt_all(); - tg.join_all(); -} +static const unsigned int QUEUE_BATCH_SIZE = 128; // This Benchmark tests the CheckQueue with a slightly realistic workload, // where checks all contain a prevector that is indirect 50% of the time @@ -99,5 +59,4 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) tg.interrupt_all(); tg.join_all(); } -BENCHMARK(CCheckQueueSpeed); -BENCHMARK(CCheckQueueSpeedPrevectorJob); +BENCHMARK(CCheckQueueSpeedPrevectorJob, 1400); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index f8956508f6..06d2abeac6 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "wallet/wallet.h" +#include <bench/bench.h> +#include <wallet/wallet.h> #include <set> @@ -56,4 +56,4 @@ static void CoinSelection(benchmark::State& state) } } -BENCHMARK(CoinSelection); +BENCHMARK(CoinSelection, 650); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 410a08e512..adb69bc6c3 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -1,19 +1,19 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <iostream> -#include "bench.h" -#include "bloom.h" -#include "hash.h" -#include "random.h" -#include "uint256.h" -#include "utiltime.h" -#include "crypto/ripemd160.h" -#include "crypto/sha1.h" -#include "crypto/sha256.h" -#include "crypto/sha512.h" +#include <bench/bench.h> +#include <bloom.h> +#include <hash.h> +#include <random.h> +#include <uint256.h> +#include <utiltime.h> +#include <crypto/ripemd160.h> +#include <crypto/sha1.h> +#include <crypto/sha256.h> +#include <crypto/sha512.h> /* Number of bytes to hash per iteration */ static const uint64_t BUFFER_SIZE = 1000*1000; @@ -46,9 +46,9 @@ static void SHA256_32b(benchmark::State& state) { std::vector<uint8_t> in(32,0); while (state.KeepRunning()) { - for (int i = 0; i < 1000000; i++) { - CSHA256().Write(in.data(), in.size()).Finalize(in.data()); - } + CSHA256() + .Write(in.data(), in.size()) + .Finalize(in.data()); } } @@ -63,10 +63,9 @@ static void SHA512(benchmark::State& state) static void SipHash_32b(benchmark::State& state) { uint256 x; + uint64_t k1 = 0; while (state.KeepRunning()) { - for (int i = 0; i < 1000000; i++) { - *((uint64_t*)x.begin()) = SipHashUint256(0, i, x); - } + *((uint64_t*)x.begin()) = SipHashUint256(0, ++k1, x); } } @@ -75,9 +74,7 @@ static void FastRandom_32bit(benchmark::State& state) FastRandomContext rng(true); uint32_t x = 0; while (state.KeepRunning()) { - for (int i = 0; i < 1000000; i++) { - x += rng.rand32(); - } + x += rng.rand32(); } } @@ -86,18 +83,16 @@ static void FastRandom_1bit(benchmark::State& state) FastRandomContext rng(true); uint32_t x = 0; while (state.KeepRunning()) { - for (int i = 0; i < 1000000; i++) { - x += rng.randbool(); - } + x += rng.randbool(); } } -BENCHMARK(RIPEMD160); -BENCHMARK(SHA1); -BENCHMARK(SHA256); -BENCHMARK(SHA512); +BENCHMARK(RIPEMD160, 440); +BENCHMARK(SHA1, 570); +BENCHMARK(SHA256, 340); +BENCHMARK(SHA512, 330); -BENCHMARK(SHA256_32b); -BENCHMARK(SipHash_32b); -BENCHMARK(FastRandom_32bit); -BENCHMARK(FastRandom_1bit); +BENCHMARK(SHA256_32b, 4700 * 1000); +BENCHMARK(SipHash_32b, 40 * 1000 * 1000); +BENCHMARK(FastRandom_32bit, 110 * 1000 * 1000); +BENCHMARK(FastRandom_1bit, 440 * 1000 * 1000); diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp index c6a05567be..ca30d81e59 100644 --- a/src/bench/lockedpool.cpp +++ b/src/bench/lockedpool.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" +#include <bench/bench.h> -#include "support/lockedpool.h" +#include <support/lockedpool.h> #include <iostream> #include <vector> @@ -43,5 +43,4 @@ static void BenchLockedPool(benchmark::State& state) addr.clear(); } -BENCHMARK(BenchLockedPool); - +BENCHMARK(BenchLockedPool, 530); diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 073bbde016..cdda0bd9be 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "policy/policy.h" -#include "txmempool.h" +#include <bench/bench.h> +#include <policy/policy.h> +#include <txmempool.h> #include <list> #include <vector> @@ -111,4 +111,4 @@ static void MempoolEviction(benchmark::State& state) } } -BENCHMARK(MempoolEviction); +BENCHMARK(MempoolEviction, 41000); diff --git a/src/bench/perf.cpp b/src/bench/perf.cpp index a549ec29ea..f92d08c56e 100644 --- a/src/bench/perf.cpp +++ b/src/bench/perf.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "perf.h" +#include <bench/perf.h> #if defined(__i386__) || defined(__x86_64__) diff --git a/src/bench/prevector_destructor.cpp b/src/bench/prevector_destructor.cpp index 55af3de4fe..39d0ee5eb1 100644 --- a/src/bench/prevector_destructor.cpp +++ b/src/bench/prevector_destructor.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "prevector.h" +#include <bench/bench.h> +#include <prevector.h> static void PrevectorDestructor(benchmark::State& state) { @@ -32,5 +32,5 @@ static void PrevectorClear(benchmark::State& state) } } -BENCHMARK(PrevectorDestructor); -BENCHMARK(PrevectorClear); +BENCHMARK(PrevectorDestructor, 5700); +BENCHMARK(PrevectorClear, 5600); diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 73c02cf718..f7f72605d7 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -1,20 +1,17 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <iostream> -#include "bench.h" -#include "bloom.h" -#include "utiltime.h" +#include <bench/bench.h> +#include <bloom.h> static void RollingBloom(benchmark::State& state) { CRollingBloomFilter filter(120000, 0.000001); std::vector<unsigned char> data(32); uint32_t count = 0; - uint32_t nEntriesPerGeneration = (120000 + 1) / 2; - uint32_t countnow = 0; uint64_t match = 0; while (state.KeepRunning()) { count++; @@ -22,16 +19,8 @@ static void RollingBloom(benchmark::State& state) data[1] = count >> 8; data[2] = count >> 16; data[3] = count >> 24; - if (countnow == nEntriesPerGeneration) { - int64_t b = GetTimeMicros(); - filter.insert(data); - int64_t e = GetTimeMicros(); - std::cout << "RollingBloom-refresh,1," << (e-b)*0.000001 << "," << (e-b)*0.000001 << "," << (e-b)*0.000001 << "\n"; - countnow = 0; - } else { - filter.insert(data); - } - countnow++; + filter.insert(data); + data[0] = count >> 24; data[1] = count >> 16; data[2] = count >> 8; @@ -40,4 +29,4 @@ static void RollingBloom(benchmark::State& state) } } -BENCHMARK(RollingBloom); +BENCHMARK(RollingBloom, 1500 * 1000); diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index ef7381c120..29dedeef0b 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" -#include "key.h" +#include <bench/bench.h> +#include <key.h> #if defined(HAVE_CONSENSUS_LIB) -#include "script/bitcoinconsensus.h" +#include <script/bitcoinconsensus.h> #endif -#include "script/script.h" -#include "script/sign.h" -#include "streams.h" +#include <script/script.h> +#include <script/sign.h> +#include <streams.h> #include <array> @@ -105,4 +105,4 @@ static void VerifyScriptBench(benchmark::State& state) } } -BENCHMARK(VerifyScriptBench); +BENCHMARK(VerifyScriptBench, 6300); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index e21a269221..e9ef75eb7b 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -1,25 +1,25 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "chainparamsbase.h" -#include "clientversion.h" -#include "fs.h" -#include "rpc/client.h" -#include "rpc/protocol.h" -#include "util.h" -#include "utilstrencodings.h" +#include <chainparamsbase.h> +#include <clientversion.h> +#include <fs.h> +#include <rpc/client.h> +#include <rpc/protocol.h> +#include <util.h> +#include <utilstrencodings.h> #include <stdio.h> #include <event2/buffer.h> #include <event2/keyvalq_struct.h> -#include "support/events.h" +#include <support/events.h> #include <univalue.h> @@ -213,6 +213,9 @@ public: /** Create a simulated `getinfo` request. */ UniValue PrepareRequest(const std::string& method, const std::vector<std::string>& args) override { + if (!args.empty()) { + throw std::runtime_error("-getinfo takes no arguments"); + } UniValue result(UniValue::VARR); result.push_back(JSONRPCRequestObj("getnetworkinfo", NullUniValue, ID_NETWORKINFO)); result.push_back(JSONRPCRequestObj("getblockchaininfo", NullUniValue, ID_BLOCKCHAININFO)); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index e4f44435ba..9bcf3fe8dd 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -1,26 +1,26 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "base58.h" -#include "clientversion.h" -#include "coins.h" -#include "consensus/consensus.h" -#include "core_io.h" -#include "keystore.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "primitives/transaction.h" -#include "script/script.h" -#include "script/sign.h" +#include <base58.h> +#include <clientversion.h> +#include <coins.h> +#include <consensus/consensus.h> +#include <core_io.h> +#include <keystore.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <script/sign.h> #include <univalue.h> -#include "util.h" -#include "utilmoneystr.h" -#include "utilstrencodings.h" +#include <util.h> +#include <utilmoneystr.h> +#include <utilstrencodings.h> #include <stdio.h> @@ -387,6 +387,10 @@ static void MutateTxAddOutMultiSig(CMutableTransaction& tx, const std::string& s scriptPubKey = GetScriptForWitness(scriptPubKey); } if (bScriptHash) { + if (scriptPubKey.size() > MAX_SCRIPT_ELEMENT_SIZE) { + throw std::runtime_error(strprintf( + "redeemScript exceeds size limit: %d > %d", scriptPubKey.size(), MAX_SCRIPT_ELEMENT_SIZE)); + } // Get the ID for the script, and then construct a P2SH destination for it. scriptPubKey = GetScriptForDestination(CScriptID(scriptPubKey)); } @@ -447,10 +451,19 @@ static void MutateTxAddOutScript(CMutableTransaction& tx, const std::string& str bScriptHash = (flags.find("S") != std::string::npos); } + if (scriptPubKey.size() > MAX_SCRIPT_SIZE) { + throw std::runtime_error(strprintf( + "script exceeds size limit: %d > %d", scriptPubKey.size(), MAX_SCRIPT_SIZE)); + } + if (bSegWit) { scriptPubKey = GetScriptForWitness(scriptPubKey); } if (bScriptHash) { + if (scriptPubKey.size() > MAX_SCRIPT_ELEMENT_SIZE) { + throw std::runtime_error(strprintf( + "redeemScript exceeds size limit: %d > %d", scriptPubKey.size(), MAX_SCRIPT_ELEMENT_SIZE)); + } scriptPubKey = GetScriptForDestination(CScriptID(scriptPubKey)); } @@ -690,10 +703,10 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, else if (command == "outaddr") MutateTxAddOutAddr(tx, commandVal); else if (command == "outpubkey") { - if (!ecc) { ecc.reset(new Secp256k1Init()); } + ecc.reset(new Secp256k1Init()); MutateTxAddOutPubKey(tx, commandVal); } else if (command == "outmultisig") { - if (!ecc) { ecc.reset(new Secp256k1Init()); } + ecc.reset(new Secp256k1Init()); MutateTxAddOutMultiSig(tx, commandVal); } else if (command == "outscript") MutateTxAddOutScript(tx, commandVal); @@ -701,7 +714,7 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, MutateTxAddOutData(tx, commandVal); else if (command == "sign") { - if (!ecc) { ecc.reset(new Secp256k1Init()); } + ecc.reset(new Secp256k1Init()); MutateTxSign(tx, commandVal); } diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 543eba0e69..de19787a16 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -1,24 +1,24 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "chainparams.h" -#include "clientversion.h" -#include "compat.h" -#include "fs.h" -#include "rpc/server.h" -#include "init.h" -#include "noui.h" -#include "scheduler.h" -#include "util.h" -#include "httpserver.h" -#include "httprpc.h" -#include "utilstrencodings.h" +#include <chainparams.h> +#include <clientversion.h> +#include <compat.h> +#include <fs.h> +#include <rpc/server.h> +#include <init.h> +#include <noui.h> +#include <scheduler.h> +#include <util.h> +#include <httpserver.h> +#include <httprpc.h> +#include <utilstrencodings.h> #include <boost/thread.hpp> @@ -120,7 +120,7 @@ bool AppInit(int argc, char* argv[]) for (int i = 1; i < argc; i++) { if (!IsSwitchChar(argv[i][0])) { fprintf(stderr, "Error: Command line contains unexpected token '%s', see bitcoind -h for a list of options.\n", argv[i]); - exit(EXIT_FAILURE); + return false; } } @@ -132,17 +132,17 @@ bool AppInit(int argc, char* argv[]) if (!AppInitBasicSetup()) { // InitError will have been called with detailed error, which ends up on console - exit(EXIT_FAILURE); + return false; } if (!AppInitParameterInteraction()) { // InitError will have been called with detailed error, which ends up on console - exit(EXIT_FAILURE); + return false; } if (!AppInitSanityChecks()) { // InitError will have been called with detailed error, which ends up on console - exit(EXIT_FAILURE); + return false; } if (gArgs.GetBoolArg("-daemon", false)) { @@ -163,7 +163,7 @@ bool AppInit(int argc, char* argv[]) if (!AppInitLockDataDirectory()) { // If locking the data directory failed, exit immediately - exit(EXIT_FAILURE); + return false; } fRet = AppInitMain(threadGroup, scheduler); } diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 6f27b7b9dc..391db0cfcb 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "blockencodings.h" -#include "consensus/consensus.h" -#include "consensus/validation.h" -#include "chainparams.h" -#include "hash.h" -#include "random.h" -#include "streams.h" -#include "txmempool.h" -#include "validation.h" -#include "util.h" +#include <blockencodings.h> +#include <consensus/consensus.h> +#include <consensus/validation.h> +#include <chainparams.h> +#include <hash.h> +#include <random.h> +#include <streams.h> +#include <txmempool.h> +#include <validation.h> +#include <util.h> #include <unordered_map> diff --git a/src/blockencodings.h b/src/blockencodings.h index 50478f9f32..ba8c1d6a2a 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -1,11 +1,11 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BLOCK_ENCODINGS_H #define BITCOIN_BLOCK_ENCODINGS_H -#include "primitives/block.h" +#include <primitives/block.h> #include <memory> diff --git a/src/bloom.cpp b/src/bloom.cpp index fa884f0bf3..f07b5b6066 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bloom.h" +#include <bloom.h> -#include "primitives/transaction.h" -#include "hash.h" -#include "script/script.h" -#include "script/standard.h" -#include "random.h" -#include "streams.h" +#include <primitives/transaction.h> +#include <hash.h> +#include <script/script.h> +#include <script/standard.h> +#include <random.h> +#include <streams.h> #include <math.h> #include <stdlib.h> diff --git a/src/bloom.h b/src/bloom.h index 7ca9682239..2f9d2109cd 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BLOOM_H #define BITCOIN_BLOOM_H -#include "serialize.h" +#include <serialize.h> #include <vector> diff --git a/src/chain.cpp b/src/chain.cpp index 5e3dd9b31b..79e8bdfa44 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -1,9 +1,9 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" +#include <chain.h> /** * CChain implementation @@ -80,12 +80,13 @@ int static inline GetSkipHeight(int height) { return (height & 1) ? InvertLowestOne(InvertLowestOne(height - 1)) + 1 : InvertLowestOne(height); } -CBlockIndex* CBlockIndex::GetAncestor(int height) +const CBlockIndex* CBlockIndex::GetAncestor(int height) const { - if (height > nHeight || height < 0) + if (height > nHeight || height < 0) { return nullptr; + } - CBlockIndex* pindexWalk = this; + const CBlockIndex* pindexWalk = this; int heightWalk = nHeight; while (heightWalk > height) { int heightSkip = GetSkipHeight(heightWalk); @@ -106,9 +107,9 @@ CBlockIndex* CBlockIndex::GetAncestor(int height) return pindexWalk; } -const CBlockIndex* CBlockIndex::GetAncestor(int height) const +CBlockIndex* CBlockIndex::GetAncestor(int height) { - return const_cast<CBlockIndex*>(this)->GetAncestor(height); + return const_cast<CBlockIndex*>(static_cast<const CBlockIndex*>(this)->GetAncestor(height)); } void CBlockIndex::BuildSkip() diff --git a/src/chain.h b/src/chain.h index f1036e5d92..3728f768c4 100644 --- a/src/chain.h +++ b/src/chain.h @@ -1,16 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CHAIN_H #define BITCOIN_CHAIN_H -#include "arith_uint256.h" -#include "primitives/block.h" -#include "pow.h" -#include "tinyformat.h" -#include "uint256.h" +#include <arith_uint256.h> +#include <primitives/block.h> +#include <pow.h> +#include <tinyformat.h> +#include <uint256.h> #include <vector> @@ -304,7 +304,7 @@ public: return (int64_t)nTimeMax; } - enum { nMedianTimeSpan=11 }; + static constexpr int nMedianTimeSpan = 11; int64_t GetMedianTimePast() const { diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 85c9cd6934..97a6c346ab 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -1,18 +1,18 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chainparams.h" -#include "consensus/merkle.h" +#include <chainparams.h> +#include <consensus/merkle.h> -#include "tinyformat.h" -#include "util.h" -#include "utilstrencodings.h" +#include <tinyformat.h> +#include <util.h> +#include <utilstrencodings.h> #include <assert.h> -#include "chainparamsseeds.h" +#include <chainparamsseeds.h> static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesisOutputScript, uint32_t nTime, uint32_t nNonce, uint32_t nBits, int32_t nVersion, const CAmount& genesisReward) { @@ -75,6 +75,7 @@ public: CMainParams() { strNetworkID = "main"; consensus.nSubsidyHalvingInterval = 210000; + consensus.BIP16Height = 173805; // 00000000000000ce80a7e057163a4db1d5ad7b20fb6f598c9597b9665c8fb0d4 - April 1, 2012 consensus.BIP34Height = 227931; consensus.BIP34Hash = uint256S("0x000000000000024b89b42a942fe0d9fea3bb44ab7bd1b19115dd6a759c0808b8"); consensus.BIP65Height = 388381; // 000000000000000004c2b624ed5d7756c508d90fd0da2c7c679febfa6c4735f0 @@ -145,7 +146,7 @@ public: fRequireStandard = true; fMineBlocksOnDemand = false; - checkpointData = (CCheckpointData) { + checkpointData = { { { 11111, uint256S("0x0000000069e244f73d78e8fd29ba2fd2ed618bd6fa2ee92559f542fdb26e7c1d")}, { 33333, uint256S("0x000000002dd5588a74784eaa7ab0507a18ad16a236e7b1ce69f00d7ddfb5d0a6")}, @@ -181,6 +182,7 @@ public: CTestNetParams() { strNetworkID = "test"; consensus.nSubsidyHalvingInterval = 210000; + consensus.BIP16Height = 514; // 00000000040b4e986385315e14bee30ad876d8b47f748025b26683116d21aa65 consensus.BIP34Height = 21111; consensus.BIP34Hash = uint256S("0x0000000023b3a96d3484e5abb3755c413e7d41500f8e2a5c3f0dd01299cd8ef8"); consensus.BIP65Height = 581885; // 00000000007f6655f22f98e72ed80d8b06dc761d5da09df0fa1dc4be4f861eb6 @@ -229,8 +231,8 @@ public: // nodes with support for servicebits filtering should be at the top vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch", true); vSeeds.emplace_back("seed.tbtc.petertodd.org", true); + vSeeds.emplace_back("seed.testnet.bitcoin.sprovoost.nl", true); vSeeds.emplace_back("testnet-seed.bluematt.me", false); - vSeeds.emplace_back("testnet-seed.bitcoin.schildbach.de", false); base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,111); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,196); @@ -247,7 +249,7 @@ public: fMineBlocksOnDemand = false; - checkpointData = (CCheckpointData) { + checkpointData = { { {546, uint256S("000000002a936ca763904c3c35fce2f3556c559c0214345d31b1bcebf76acb70")}, } @@ -271,6 +273,7 @@ public: CRegTestParams() { strNetworkID = "regtest"; consensus.nSubsidyHalvingInterval = 150; + consensus.BIP16Height = 0; // always enforce P2SH BIP16 on regtest consensus.BIP34Height = 100000000; // BIP34 has not activated on regtest (far in the future so block v1 are not rejected in tests) consensus.BIP34Hash = uint256(); consensus.BIP65Height = 1351; // BIP65 activated on regtest (Used in rpc activation tests) @@ -284,13 +287,13 @@ public: consensus.nMinerConfirmationWindow = 144; // Faster than normal for regtest (144 instead of 2016) consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].bit = 28; consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nStartTime = 0; - consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = 999999999999ULL; + consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT; consensus.vDeployments[Consensus::DEPLOYMENT_CSV].bit = 0; consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nStartTime = 0; - consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = 999999999999ULL; + consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT; consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].bit = 1; - consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 0; - consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 999999999999ULL; + consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = Consensus::BIP9Deployment::ALWAYS_ACTIVE; + consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = Consensus::BIP9Deployment::NO_TIMEOUT; // The best chain should have at least this much work. consensus.nMinimumChainWork = uint256S("0x00"); @@ -317,7 +320,7 @@ public: fRequireStandard = false; fMineBlocksOnDemand = true; - checkpointData = (CCheckpointData) { + checkpointData = { { {0, uint256S("0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206")}, } diff --git a/src/chainparams.h b/src/chainparams.h index 3948c9163f..ecfe1a0ac3 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -1,15 +1,15 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CHAINPARAMS_H #define BITCOIN_CHAINPARAMS_H -#include "chainparamsbase.h" -#include "consensus/params.h" -#include "primitives/block.h" -#include "protocol.h" +#include <chainparamsbase.h> +#include <consensus/params.h> +#include <primitives/block.h> +#include <protocol.h> #include <memory> #include <vector> diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index c966683b72..a03137f407 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -1,12 +1,12 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chainparamsbase.h" +#include <chainparamsbase.h> -#include "tinyformat.h" -#include "util.h" +#include <tinyformat.h> +#include <util.h> #include <assert.h> diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index fc101f5b77..b4d2bb4f08 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2015 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp index 5afe3e66b6..9189c9a8ad 100644 --- a/src/checkpoints.cpp +++ b/src/checkpoints.cpp @@ -1,14 +1,13 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "checkpoints.h" +#include <checkpoints.h> -#include "chain.h" -#include "chainparams.h" -#include "reverse_iterator.h" -#include "validation.h" -#include "uint256.h" +#include <chain.h> +#include <chainparams.h> +#include <reverse_iterator.h> +#include <validation.h> #include <stdint.h> diff --git a/src/checkpoints.h b/src/checkpoints.h index 7449d1063b..bf935f80a7 100644 --- a/src/checkpoints.h +++ b/src/checkpoints.h @@ -1,11 +1,11 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CHECKPOINTS_H #define BITCOIN_CHECKPOINTS_H -#include "uint256.h" +#include <uint256.h> #include <map> diff --git a/src/checkqueue.h b/src/checkqueue.h index 6377fbe942..9b4a460bae 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2015 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CHECKQUEUE_H #define BITCOIN_CHECKQUEUE_H -#include "sync.h" +#include <sync.h> #include <algorithm> #include <vector> diff --git a/src/clientversion.cpp b/src/clientversion.cpp index 8a4b408831..7aa0a8d660 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -1,12 +1,11 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "clientversion.h" +#include <clientversion.h> -#include "tinyformat.h" +#include <tinyformat.h> -#include <string> /** * Name of client reported in the 'version' message. Report the same name @@ -39,7 +38,7 @@ const std::string CLIENT_NAME("Satoshi"); //! First, include build.h if requested #ifdef HAVE_BUILD_INFO -#include "build.h" +#include <obj/build.h> #endif //! git will put "#define GIT_ARCHIVE 1" on the next line inside archives. $Format:%n#define GIT_ARCHIVE 1$ diff --git a/src/clientversion.h b/src/clientversion.h index 3d5392619b..d7321fad86 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #define BITCOIN_CLIENTVERSION_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif //HAVE_CONFIG_H // Check that required client information is defined diff --git a/src/coins.cpp b/src/coins.cpp index b45dfc3342..8dfb35c2eb 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -1,14 +1,11 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "coins.h" +#include <coins.h> -#include "consensus/consensus.h" -#include "memusage.h" -#include "random.h" - -#include <assert.h> +#include <consensus/consensus.h> +#include <random.h> bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } @@ -146,56 +143,58 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { } bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { - for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { - if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Ignore non-dirty entries (optimization). - CCoinsMap::iterator itUs = cacheCoins.find(it->first); - if (itUs == cacheCoins.end()) { - // The parent cache does not have an entry, while the child does - // We can ignore it if it's both FRESH and pruned in the child - if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { - // Otherwise we will need to create it in the parent - // and move the data up and mark it as dirty - CCoinsCacheEntry& entry = cacheCoins[it->first]; - entry.coin = std::move(it->second.coin); - cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); - entry.flags = CCoinsCacheEntry::DIRTY; - // We can mark it FRESH in the parent if it was FRESH in the child - // Otherwise it might have just been flushed from the parent's cache - // and already exist in the grandparent - if (it->second.flags & CCoinsCacheEntry::FRESH) - entry.flags |= CCoinsCacheEntry::FRESH; + for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); it = mapCoins.erase(it)) { + // Ignore non-dirty entries (optimization). + if (!(it->second.flags & CCoinsCacheEntry::DIRTY)) { + continue; + } + CCoinsMap::iterator itUs = cacheCoins.find(it->first); + if (itUs == cacheCoins.end()) { + // The parent cache does not have an entry, while the child does + // We can ignore it if it's both FRESH and pruned in the child + if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { + // Otherwise we will need to create it in the parent + // and move the data up and mark it as dirty + CCoinsCacheEntry& entry = cacheCoins[it->first]; + entry.coin = std::move(it->second.coin); + cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); + entry.flags = CCoinsCacheEntry::DIRTY; + // We can mark it FRESH in the parent if it was FRESH in the child + // Otherwise it might have just been flushed from the parent's cache + // and already exist in the grandparent + if (it->second.flags & CCoinsCacheEntry::FRESH) { + entry.flags |= CCoinsCacheEntry::FRESH; } - } else { - // Assert that the child cache entry was not marked FRESH if the - // parent cache entry has unspent outputs. If this ever happens, - // it means the FRESH flag was misapplied and there is a logic - // error in the calling code. - if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) - throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); + } + } else { + // Assert that the child cache entry was not marked FRESH if the + // parent cache entry has unspent outputs. If this ever happens, + // it means the FRESH flag was misapplied and there is a logic + // error in the calling code. + if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) { + throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); + } - // Found the entry in the parent cache - if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { - // The grandparent does not have an entry, and the child is - // modified and being pruned. This means we can just delete - // it from the parent. - cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); - cacheCoins.erase(itUs); - } else { - // A normal modification. - cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); - itUs->second.coin = std::move(it->second.coin); - cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); - itUs->second.flags |= CCoinsCacheEntry::DIRTY; - // NOTE: It is possible the child has a FRESH flag here in - // the event the entry we found in the parent is pruned. But - // we must not copy that FRESH flag to the parent as that - // pruned state likely still needs to be communicated to the - // grandparent. - } + // Found the entry in the parent cache + if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { + // The grandparent does not have an entry, and the child is + // modified and being pruned. This means we can just delete + // it from the parent. + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + cacheCoins.erase(itUs); + } else { + // A normal modification. + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + itUs->second.coin = std::move(it->second.coin); + cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); + itUs->second.flags |= CCoinsCacheEntry::DIRTY; + // NOTE: It is possible the child has a FRESH flag here in + // the event the entry we found in the parent is pruned. But + // we must not copy that FRESH flag to the parent as that + // pruned state likely still needs to be communicated to the + // grandparent. } } - CCoinsMap::iterator itOld = it++; - mapCoins.erase(itOld); } hashBlock = hashBlockIn; return true; diff --git a/src/coins.h b/src/coins.h index 181b2fd4b9..c6850947e2 100644 --- a/src/coins.h +++ b/src/coins.h @@ -1,18 +1,18 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H -#include "primitives/transaction.h" -#include "compressor.h" -#include "core_memusage.h" -#include "hash.h" -#include "memusage.h" -#include "serialize.h" -#include "uint256.h" +#include <primitives/transaction.h> +#include <compressor.h> +#include <core_memusage.h> +#include <hash.h> +#include <memusage.h> +#include <serialize.h> +#include <uint256.h> #include <assert.h> #include <stdint.h> diff --git a/src/compat.h b/src/compat.h index e022659c01..aae84b1181 100644 --- a/src/compat.h +++ b/src/compat.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,7 +7,7 @@ #define BITCOIN_COMPAT_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #ifdef WIN32 @@ -31,8 +31,9 @@ #include <mswsock.h> #include <windows.h> #include <ws2tcpip.h> +#include <stdint.h> #else -#include <sys/fcntl.h> +#include <fcntl.h> #include <sys/mman.h> #include <sys/select.h> #include <sys/socket.h> @@ -49,7 +50,7 @@ #ifndef WIN32 typedef unsigned int SOCKET; -#include "errno.h" +#include <errno.h> #define WSAGetLastError() errno #define WSAEINVAL EINVAL #define WSAEALREADY EALREADY @@ -71,6 +72,15 @@ typedef unsigned int SOCKET; #else #define MAX_PATH 1024 #endif +#ifdef _MSC_VER +#if !defined(ssize_t) +#ifdef _WIN64 +typedef int64_t ssize_t; +#else +typedef int32_t ssize_t; +#endif +#endif +#endif #if HAVE_DECL_STRNLEN == 0 size_t strnlen( const char *start, size_t max_len); diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index d93ff7413a..c331af2b5e 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #define BITCOIN_COMPAT_BYTESWAP_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <stdint.h> @@ -37,7 +37,7 @@ inline uint16_t bswap_16(uint16_t x) { return (x >> 8) | (x << 8); } -#endif // HAVE_DECL_BSWAP16 +#endif // HAVE_DECL_BSWAP16 == 0 #if HAVE_DECL_BSWAP_32 == 0 inline uint32_t bswap_32(uint32_t x) @@ -45,7 +45,7 @@ inline uint32_t bswap_32(uint32_t x) return (((x & 0xff000000U) >> 24) | ((x & 0x00ff0000U) >> 8) | ((x & 0x0000ff00U) << 8) | ((x & 0x000000ffU) << 24)); } -#endif // HAVE_DECL_BSWAP32 +#endif // HAVE_DECL_BSWAP32 == 0 #if HAVE_DECL_BSWAP_64 == 0 inline uint64_t bswap_64(uint64_t x) @@ -59,7 +59,7 @@ inline uint64_t bswap_64(uint64_t x) | ((x & 0x000000000000ff00ull) << 40) | ((x & 0x00000000000000ffull) << 56)); } -#endif // HAVE_DECL_BSWAP64 +#endif // HAVE_DECL_BSWAP64 == 0 #endif // defined(__APPLE__) diff --git a/src/compat/endian.h b/src/compat/endian.h index dbf178f53c..e5c7e50223 100644 --- a/src/compat/endian.h +++ b/src/compat/endian.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,10 +6,10 @@ #define BITCOIN_COMPAT_ENDIAN_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "compat/byteswap.h" +#include <compat/byteswap.h> #include <stdint.h> diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp index 3b9c70df7f..55da5ef63f 100644 --- a/src/compat/glibc_compat.cpp +++ b/src/compat/glibc_compat.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <cstddef> diff --git a/src/compat/glibc_sanity.cpp b/src/compat/glibc_sanity.cpp index b4d1c90992..1c22dc073c 100644 --- a/src/compat/glibc_sanity.cpp +++ b/src/compat/glibc_sanity.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <cstddef> diff --git a/src/compat/glibcxx_sanity.cpp b/src/compat/glibcxx_sanity.cpp index 569fb1bbe8..257665a061 100644 --- a/src/compat/glibcxx_sanity.cpp +++ b/src/compat/glibcxx_sanity.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/compat/strnlen.cpp b/src/compat/strnlen.cpp index 1ac266c2d1..f7344d39f5 100644 --- a/src/compat/strnlen.cpp +++ b/src/compat/strnlen.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <cstring> diff --git a/src/compressor.cpp b/src/compressor.cpp index f4c12f38d2..86de2900e9 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "compressor.h" +#include <compressor.h> -#include "hash.h" -#include "pubkey.h" -#include "script/standard.h" +#include <hash.h> +#include <pubkey.h> +#include <script/standard.h> bool CScriptCompressor::IsToKeyID(CKeyID &hash) const { diff --git a/src/compressor.h b/src/compressor.h index 094c1bcfe1..ee26f4c533 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -1,14 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_COMPRESSOR_H #define BITCOIN_COMPRESSOR_H -#include "primitives/transaction.h" -#include "script/script.h" -#include "serialize.h" +#include <primitives/transaction.h> +#include <script/script.h> +#include <serialize.h> class CKeyID; class CPubKey; diff --git a/src/consensus/consensus.h b/src/consensus/consensus.h index ddd4ee9fab..650635a766 100644 --- a/src/consensus/consensus.h +++ b/src/consensus/consensus.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -24,12 +24,9 @@ static const size_t MIN_TRANSACTION_WEIGHT = WITNESS_SCALE_FACTOR * 60; // 60 is static const size_t MIN_SERIALIZABLE_TRANSACTION_WEIGHT = WITNESS_SCALE_FACTOR * 10; // 10 is the lower bound for the size of a serialized CTransaction /** Flags for nSequence and nLockTime locks */ -enum { - /* Interpret sequence numbers as relative lock-time constraints. */ - LOCKTIME_VERIFY_SEQUENCE = (1 << 0), - - /* Use GetMedianTimePast() instead of nTime for end point timestamp. */ - LOCKTIME_MEDIAN_TIME_PAST = (1 << 1), -}; +/** Interpret sequence numbers as relative lock-time constraints. */ +static constexpr unsigned int LOCKTIME_VERIFY_SEQUENCE = (1 << 0); +/** Use GetMedianTimePast() instead of nTime for end point timestamp. */ +static constexpr unsigned int LOCKTIME_MEDIAN_TIME_PAST = (1 << 1); #endif // BITCOIN_CONSENSUS_CONSENSUS_H diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index 798ce4b5fd..74a9ebb2e3 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "merkle.h" -#include "hash.h" -#include "utilstrencodings.h" +#include <consensus/merkle.h> +#include <hash.h> +#include <utilstrencodings.h> /* WARNING! If you're reading this because you're learning about crypto and/or designing a new system that will use merkle trees, keep in mind diff --git a/src/consensus/merkle.h b/src/consensus/merkle.h index 33764c7460..d57bb3412e 100644 --- a/src/consensus/merkle.h +++ b/src/consensus/merkle.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,9 +8,9 @@ #include <stdint.h> #include <vector> -#include "primitives/transaction.h" -#include "primitives/block.h" -#include "uint256.h" +#include <primitives/transaction.h> +#include <primitives/block.h> +#include <uint256.h> uint256 ComputeMerkleRoot(const std::vector<uint256>& leaves, bool* mutated = nullptr); std::vector<uint256> ComputeMerkleBranch(const std::vector<uint256>& leaves, uint32_t position); diff --git a/src/consensus/params.h b/src/consensus/params.h index 6240e82857..4ef808c856 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -1,12 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CONSENSUS_PARAMS_H #define BITCOIN_CONSENSUS_PARAMS_H -#include "uint256.h" +#include <uint256.h> +#include <limits> #include <map> #include <string> @@ -31,6 +32,15 @@ struct BIP9Deployment { int64_t nStartTime; /** Timeout/expiry MedianTime for the deployment attempt. */ int64_t nTimeout; + + /** Constant for nTimeout very far in the future. */ + static constexpr int64_t NO_TIMEOUT = std::numeric_limits<int64_t>::max(); + + /** Special value for nStartTime indicating that the deployment is always active. + * This is useful for testing, as it means tests don't need to deal with the activation + * process (which takes at least 3 BIP9 intervals). Only tests that specifically test the + * behaviour during activation cannot use this. */ + static constexpr int64_t ALWAYS_ACTIVE = -1; }; /** @@ -39,6 +49,8 @@ struct BIP9Deployment { struct Params { uint256 hashGenesisBlock; int nSubsidyHalvingInterval; + /** Block height at which BIP16 becomes active */ + int BIP16Height; /** Block height and hash at which BIP34 becomes active */ int BIP34Height; uint256 BIP34Hash; diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 0a71915d1d..be73d0a2f9 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -2,18 +2,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "tx_verify.h" +#include <consensus/tx_verify.h> -#include "consensus.h" -#include "primitives/transaction.h" -#include "script/interpreter.h" -#include "validation.h" +#include <consensus/consensus.h> +#include <primitives/transaction.h> +#include <script/interpreter.h> +#include <consensus/validation.h> // TODO remove the following dependencies -#include "chain.h" -#include "coins.h" -#include "utilmoneystr.h" - +#include <chain.h> +#include <coins.h> +#include <utilmoneystr.h> + bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) { if (tx.nLockTime == 0) @@ -205,46 +205,46 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fChe return true; } -bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight) +bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee) { - // This doesn't trigger the DoS code on purpose; if it did, it would make it easier - // for an attacker to attempt to split the network. - if (!inputs.HaveInputs(tx)) - return state.Invalid(false, 0, "", "Inputs unavailable"); - - CAmount nValueIn = 0; - CAmount nFees = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const COutPoint &prevout = tx.vin[i].prevout; - const Coin& coin = inputs.AccessCoin(prevout); - assert(!coin.IsSpent()); - - // If prev is coinbase, check that it's matured - if (coin.IsCoinBase()) { - if (nSpendHeight - coin.nHeight < COINBASE_MATURITY) - return state.Invalid(false, - REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", - strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); - } - - // Check for negative or overflow input values - nValueIn += coin.out.nValue; - if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); + // are the actual inputs available? + if (!inputs.HaveInputs(tx)) { + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-missingorspent", false, + strprintf("%s: inputs missing/spent", __func__)); + } + + CAmount nValueIn = 0; + for (unsigned int i = 0; i < tx.vin.size(); ++i) { + const COutPoint &prevout = tx.vin[i].prevout; + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); + + // If prev is coinbase, check that it's matured + if (coin.IsCoinBase() && nSpendHeight - coin.nHeight < COINBASE_MATURITY) { + return state.Invalid(false, + REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", + strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); + } + // Check for negative or overflow input values + nValueIn += coin.out.nValue; + if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) { + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); } + } + + const CAmount value_out = tx.GetValueOut(); + if (nValueIn < value_out) { + return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, + strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(value_out))); + } + + // Tally transaction fees + const CAmount txfee_aux = nValueIn - value_out; + if (!MoneyRange(txfee_aux)) { + return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); + } - if (nValueIn < tx.GetValueOut()) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, - strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(tx.GetValueOut()))); - - // Tally transaction fees - CAmount nTxFee = nValueIn - tx.GetValueOut(); - if (nTxFee < 0) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-negative"); - nFees += nTxFee; - if (!MoneyRange(nFees)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); + txfee = txfee_aux; return true; } diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h index d46d3294ca..a4836ceda4 100644 --- a/src/consensus/tx_verify.h +++ b/src/consensus/tx_verify.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CONSENSUS_TX_VERIFY_H #define BITCOIN_CONSENSUS_TX_VERIFY_H +#include <amount.h> + #include <stdint.h> #include <vector> @@ -22,9 +24,10 @@ namespace Consensus { /** * Check whether all inputs of this transaction are valid (no double spends and amounts) * This does not modify the UTXO set. This does not check scripts and sigs. + * @param[out] txfee Set to the transaction fee if successful. * Preconditions: tx.IsCoinBase() is false. */ -bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight); +bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight, CAmount& txfee); } // namespace Consensus /** Auxiliary functions for transaction validation (ideally should not be exposed) */ diff --git a/src/consensus/validation.h b/src/consensus/validation.h index b6740c9d9f..c2a343c155 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,10 +7,10 @@ #define BITCOIN_CONSENSUS_VALIDATION_H #include <string> -#include "version.h" -#include "consensus/consensus.h" -#include "primitives/transaction.h" -#include "primitives/block.h" +#include <version.h> +#include <consensus/consensus.h> +#include <primitives/transaction.h> +#include <primitives/block.h> /** "reject" message codes */ static const unsigned char REJECT_MALFORMED = 0x01; diff --git a/src/core_io.h b/src/core_io.h index ccc72ebb32..377633ac77 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -1,11 +1,11 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CORE_IO_H #define BITCOIN_CORE_IO_H -#include "amount.h" +#include <amount.h> #include <string> #include <vector> @@ -20,7 +20,7 @@ class UniValue; // core_read.cpp CScript ParseScript(const std::string& s); std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false); -bool DecodeHexTx(CMutableTransaction& tx, const std::string& strHexTx, bool fTryNoWitness = false); +bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no_witness = false, bool try_witness = true); bool DecodeHexBlk(CBlock&, const std::string& strHexBlk); uint256 ParseHashUV(const UniValue& v, const std::string& strName); uint256 ParseHashStr(const std::string&, const std::string& strName); diff --git a/src/core_memusage.h b/src/core_memusage.h index f038e7b154..6c03938d0c 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -1,13 +1,13 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CORE_MEMUSAGE_H #define BITCOIN_CORE_MEMUSAGE_H -#include "primitives/transaction.h" -#include "primitives/block.h" -#include "memusage.h" +#include <primitives/transaction.h> +#include <primitives/block.h> +#include <memusage.h> static inline size_t RecursiveDynamicUsage(const CScript& script) { return memusage::DynamicUsage(script); diff --git a/src/core_read.cpp b/src/core_read.cpp index 7018131a13..4ccb967087 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -1,18 +1,18 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "core_io.h" +#include <core_io.h> -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "script/script.h" -#include "serialize.h" -#include "streams.h" +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <serialize.h> +#include <streams.h> #include <univalue.h> -#include "util.h" -#include "utilstrencodings.h" -#include "version.h" +#include <util.h> +#include <utilstrencodings.h> +#include <version.h> #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/predicate.hpp> @@ -108,39 +108,39 @@ bool CheckTxScriptsSanity(const CMutableTransaction& tx) return true; } -bool DecodeHexTx(CMutableTransaction& tx, const std::string& strHexTx, bool fTryNoWitness) +bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no_witness, bool try_witness) { - if (!IsHex(strHexTx)) { + if (!IsHex(hex_tx)) { return false; } - std::vector<unsigned char> txData(ParseHex(strHexTx)); + std::vector<unsigned char> txData(ParseHex(hex_tx)); - if (fTryNoWitness) { + if (try_no_witness) { CDataStream ssData(txData, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); try { ssData >> tx; - if (ssData.eof() && CheckTxScriptsSanity(tx)) { + if (ssData.eof() && (!try_witness || CheckTxScriptsSanity(tx))) { return true; } - } - catch (const std::exception&) { + } catch (const std::exception&) { // Fall through. } } - CDataStream ssData(txData, SER_NETWORK, PROTOCOL_VERSION); - try { - ssData >> tx; - if (!ssData.empty()) { - return false; + if (try_witness) { + CDataStream ssData(txData, SER_NETWORK, PROTOCOL_VERSION); + try { + ssData >> tx; + if (ssData.empty()) { + return true; + } + } catch (const std::exception&) { + // Fall through. } } - catch (const std::exception&) { - return false; - } - - return true; + + return false; } bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) diff --git a/src/core_write.cpp b/src/core_write.cpp index e16db13650..ab6918e41d 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -1,20 +1,20 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "core_io.h" +#include <core_io.h> -#include "base58.h" -#include "consensus/consensus.h" -#include "consensus/validation.h" -#include "script/script.h" -#include "script/standard.h" -#include "serialize.h" -#include "streams.h" +#include <base58.h> +#include <consensus/consensus.h> +#include <consensus/validation.h> +#include <script/script.h> +#include <script/standard.h> +#include <serialize.h> +#include <streams.h> #include <univalue.h> -#include "util.h" -#include "utilmoneystr.h" -#include "utilstrencodings.h" +#include <util.h> +#include <utilmoneystr.h> +#include <utilstrencodings.h> UniValue ValueFromAmount(const CAmount& amount) { diff --git a/src/crypto/aes.cpp b/src/crypto/aes.cpp index 5e70d25eee..bf7a252349 100644 --- a/src/crypto/aes.cpp +++ b/src/crypto/aes.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "aes.h" -#include "crypto/common.h" +#include <crypto/aes.h> +#include <crypto/common.h> #include <assert.h> #include <string.h> extern "C" { -#include "crypto/ctaes/ctaes.c" +#include <crypto/ctaes/ctaes.c> } AES128Encrypt::AES128Encrypt(const unsigned char key[16]) diff --git a/src/crypto/aes.h b/src/crypto/aes.h index a7b63b19df..2dec8d9558 100644 --- a/src/crypto/aes.h +++ b/src/crypto/aes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // @@ -8,7 +8,7 @@ #define BITCOIN_CRYPTO_AES_H extern "C" { -#include "crypto/ctaes/ctaes.h" +#include <crypto/ctaes/ctaes.h> } static const int AES_BLOCKSIZE = 16; diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index 4038ae9f86..ac4470f04f 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -5,8 +5,8 @@ // Based on the public domain implementation 'merged' by D. J. Bernstein // See https://cr.yp.to/chacha.html. -#include "crypto/common.h" -#include "crypto/chacha20.h" +#include <crypto/common.h> +#include <crypto/chacha20.h> #include <string.h> diff --git a/src/crypto/common.h b/src/crypto/common.h index bd9bc9420b..825b430978 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,13 +6,13 @@ #define BITCOIN_CRYPTO_COMMON_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <stdint.h> #include <string.h> -#include "compat/endian.h" +#include <compat/endian.h> uint16_t static inline ReadLE16(const unsigned char* ptr) { diff --git a/src/crypto/hmac_sha256.cpp b/src/crypto/hmac_sha256.cpp index 3c791625d0..d4afe1439f 100644 --- a/src/crypto/hmac_sha256.cpp +++ b/src/crypto/hmac_sha256.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/hmac_sha256.h" +#include <crypto/hmac_sha256.h> #include <string.h> diff --git a/src/crypto/hmac_sha256.h b/src/crypto/hmac_sha256.h index 8c42fcfe14..4fb30b7ac0 100644 --- a/src/crypto/hmac_sha256.h +++ b/src/crypto/hmac_sha256.h @@ -1,11 +1,11 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CRYPTO_HMAC_SHA256_H #define BITCOIN_CRYPTO_HMAC_SHA256_H -#include "crypto/sha256.h" +#include <crypto/sha256.h> #include <stdint.h> #include <stdlib.h> diff --git a/src/crypto/hmac_sha512.cpp b/src/crypto/hmac_sha512.cpp index 5939c6ec47..d9c4d04100 100644 --- a/src/crypto/hmac_sha512.cpp +++ b/src/crypto/hmac_sha512.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/hmac_sha512.h" +#include <crypto/hmac_sha512.h> #include <string.h> diff --git a/src/crypto/hmac_sha512.h b/src/crypto/hmac_sha512.h index 17dee61ea8..ab84ee7652 100644 --- a/src/crypto/hmac_sha512.h +++ b/src/crypto/hmac_sha512.h @@ -1,11 +1,11 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CRYPTO_HMAC_SHA512_H #define BITCOIN_CRYPTO_HMAC_SHA512_H -#include "crypto/sha512.h" +#include <crypto/sha512.h> #include <stdint.h> #include <stdlib.h> diff --git a/src/crypto/ripemd160.cpp b/src/crypto/ripemd160.cpp index 77c9acfc26..51468ec8d0 100644 --- a/src/crypto/ripemd160.cpp +++ b/src/crypto/ripemd160.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/ripemd160.h" +#include <crypto/ripemd160.h> -#include "crypto/common.h" +#include <crypto/common.h> #include <string.h> diff --git a/src/crypto/sha1.cpp b/src/crypto/sha1.cpp index 0b895b33a2..dc96ac507a 100644 --- a/src/crypto/sha1.cpp +++ b/src/crypto/sha1.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/sha1.h" +#include <crypto/sha1.h> -#include "crypto/common.h" +#include <crypto/common.h> #include <string.h> diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 29afe86ec7..f3245b8dea 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/sha256.h" -#include "crypto/common.h" +#include <crypto/sha256.h> +#include <crypto/common.h> #include <assert.h> #include <string.h> diff --git a/src/crypto/sha256.h b/src/crypto/sha256.h index aa4f3972cc..dd30fe396f 100644 --- a/src/crypto/sha256.h +++ b/src/crypto/sha256.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/sha512.cpp b/src/crypto/sha512.cpp index 564127cc31..dff4d8da1a 100644 --- a/src/crypto/sha512.cpp +++ b/src/crypto/sha512.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/sha512.h" +#include <crypto/sha512.h> -#include "crypto/common.h" +#include <crypto/common.h> #include <string.h> diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index dfc90f3ab9..4e1e403f69 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -1,12 +1,10 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "dbwrapper.h" +#include <dbwrapper.h> -#include "fs.h" -#include "util.h" -#include "random.h" +#include <random.h> #include <leveldb/cache.h> #include <leveldb/env.h> @@ -64,7 +62,7 @@ public: assert(p <= limit); base[std::min(bufsize - 1, (int)(p - base))] = '\0'; - LogPrintStr(base); + LogPrintf("leveldb: %s", base); if (base != buffer) { delete[] base; } diff --git a/src/dbwrapper.h b/src/dbwrapper.h index e19fde51c1..a29938ce33 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -1,17 +1,17 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_DBWRAPPER_H #define BITCOIN_DBWRAPPER_H -#include "clientversion.h" -#include "fs.h" -#include "serialize.h" -#include "streams.h" -#include "util.h" -#include "utilstrencodings.h" -#include "version.h" +#include <clientversion.h> +#include <fs.h> +#include <serialize.h> +#include <streams.h> +#include <util.h> +#include <utilstrencodings.h> +#include <version.h> #include <leveldb/db.h> #include <leveldb/write_batch.h> diff --git a/src/fs.cpp b/src/fs.cpp index a5e12f1cfc..570ed3e2ee 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -1,4 +1,4 @@ -#include "fs.h" +#include <fs.h> namespace fsbridge { diff --git a/src/hash.cpp b/src/hash.cpp index 5a15600be5..d31e094540 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -1,11 +1,10 @@ -// Copyright (c) 2013-2016 The Bitcoin Core developers +// Copyright (c) 2013-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "hash.h" -#include "crypto/common.h" -#include "crypto/hmac_sha512.h" -#include "pubkey.h" +#include <hash.h> +#include <crypto/common.h> +#include <crypto/hmac_sha512.h> inline uint32_t ROTL32(uint32_t x, int8_t r) diff --git a/src/hash.h b/src/hash.h index 474b13d65b..35995a2d15 100644 --- a/src/hash.h +++ b/src/hash.h @@ -1,17 +1,17 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_HASH_H #define BITCOIN_HASH_H -#include "crypto/ripemd160.h" -#include "crypto/sha256.h" -#include "prevector.h" -#include "serialize.h" -#include "uint256.h" -#include "version.h" +#include <crypto/ripemd160.h> +#include <crypto/sha256.h> +#include <prevector.h> +#include <serialize.h> +#include <uint256.h> +#include <version.h> #include <vector> diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 91f96ef207..66f7a6a718 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -1,20 +1,20 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "httprpc.h" - -#include "base58.h" -#include "chainparams.h" -#include "httpserver.h" -#include "rpc/protocol.h" -#include "rpc/server.h" -#include "random.h" -#include "sync.h" -#include "util.h" -#include "utilstrencodings.h" -#include "ui_interface.h" -#include "crypto/hmac_sha256.h" +#include <httprpc.h> + +#include <base58.h> +#include <chainparams.h> +#include <httpserver.h> +#include <rpc/protocol.h> +#include <rpc/server.h> +#include <random.h> +#include <sync.h> +#include <util.h> +#include <utilstrencodings.h> +#include <ui_interface.h> +#include <crypto/hmac_sha256.h> #include <stdio.h> #include <boost/algorithm/string.hpp> // boost::trim @@ -62,7 +62,7 @@ private: /* Pre-base64-encoded authentication token */ static std::string strRPCUserColonPass; /* Stored RPC timer interface (for unregistration) */ -static HTTPRPCTimerInterface* httpRPCTimerInterface = nullptr; +static std::unique_ptr<HTTPRPCTimerInterface> httpRPCTimerInterface; static void JSONErrorReply(HTTPRequest* req, const UniValue& objError, const UniValue& id) { @@ -192,7 +192,7 @@ static bool HTTPReq_JSONRPC(HTTPRequest* req, const std::string &) // array of requests } else if (valRequest.isArray()) - strReply = JSONRPCExecBatch(valRequest.get_array()); + strReply = JSONRPCExecBatch(jreq, valRequest.get_array()); else throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error"); @@ -238,8 +238,8 @@ bool StartHTTPRPC() RegisterHTTPHandler("/wallet/", false, HTTPReq_JSONRPC); #endif assert(EventBase()); - httpRPCTimerInterface = new HTTPRPCTimerInterface(EventBase()); - RPCSetTimerInterface(httpRPCTimerInterface); + httpRPCTimerInterface = MakeUnique<HTTPRPCTimerInterface>(EventBase()); + RPCSetTimerInterface(httpRPCTimerInterface.get()); return true; } @@ -253,8 +253,7 @@ void StopHTTPRPC() LogPrint(BCLog::RPC, "Stopping HTTP RPC server\n"); UnregisterHTTPHandler("/", true); if (httpRPCTimerInterface) { - RPCUnsetTimerInterface(httpRPCTimerInterface); - delete httpRPCTimerInterface; - httpRPCTimerInterface = nullptr; + RPCUnsetTimerInterface(httpRPCTimerInterface.get()); + httpRPCTimerInterface.reset(); } } diff --git a/src/httprpc.h b/src/httprpc.h index a89a8f0fbf..ccae29b45d 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 5923871691..a022d220e0 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "httpserver.h" +#include <httpserver.h> -#include "chainparamsbase.h" -#include "compat.h" -#include "util.h" -#include "utilstrencodings.h" -#include "netbase.h" -#include "rpc/protocol.h" // For HTTP status codes -#include "sync.h" -#include "ui_interface.h" +#include <chainparamsbase.h> +#include <compat.h> +#include <util.h> +#include <utilstrencodings.h> +#include <netbase.h> +#include <rpc/protocol.h> // For HTTP status codes +#include <sync.h> +#include <ui_interface.h> #include <stdio.h> #include <stdlib.h> @@ -24,10 +24,11 @@ #include <event2/thread.h> #include <event2/buffer.h> +#include <event2/bufferevent.h> #include <event2/util.h> #include <event2/keyvalq_struct.h> -#include "support/events.h" +#include <support/events.h> #ifdef EVENT__HAVE_NETINET_IN_H #include <netinet/in.h> @@ -239,6 +240,16 @@ static std::string RequestMethodString(HTTPRequest::RequestMethod m) /** HTTP request callback */ static void http_request_cb(struct evhttp_request* req, void* arg) { + // Disable reading to work around a libevent bug, fixed in 2.2.0. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { + evhttp_connection* conn = evhttp_request_get_connection(req); + if (conn) { + bufferevent* bev = evhttp_connection_get_bufferevent(conn); + if (bev) { + bufferevent_disable(bev, EV_READ); + } + } + } std::unique_ptr<HTTPRequest> hreq(new HTTPRequest(req)); LogPrint(BCLog::HTTP, "Received a %s request for %s from %s\n", @@ -481,6 +492,8 @@ void StopHTTPServer() } if (eventBase) { LogPrint(BCLog::HTTP, "Waiting for HTTP event thread to exit\n"); + // Exit the event loop as soon as there are no active events. + event_base_loopexit(eventBase, nullptr); // Give event loop a few seconds to exit (to send back last RPC responses), then break it // Before this was solved with event_base_loopexit, but that didn't work as expected in // at least libevent 2.0.21 and always introduced a delay. In libevent @@ -599,8 +612,21 @@ void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) struct evbuffer* evb = evhttp_request_get_output_buffer(req); assert(evb); evbuffer_add(evb, strReply.data(), strReply.size()); - HTTPEvent* ev = new HTTPEvent(eventBase, true, - std::bind(evhttp_send_reply, req, nStatus, (const char*)nullptr, (struct evbuffer *)nullptr)); + auto req_copy = req; + HTTPEvent* ev = new HTTPEvent(eventBase, true, [req_copy, nStatus]{ + evhttp_send_reply(req_copy, nStatus, nullptr, nullptr); + // Re-enable reading from the socket. This is the second part of the libevent + // workaround above. + if (event_get_version_number() >= 0x02010600 && event_get_version_number() < 0x02020001) { + evhttp_connection* conn = evhttp_request_get_connection(req_copy); + if (conn) { + bufferevent* bev = evhttp_connection_get_bufferevent(conn); + if (bev) { + bufferevent_enable(bev, EV_READ | EV_WRITE); + } + } + } + }); ev->trigger(nullptr); replySent = true; req = nullptr; // transferred back to main thread diff --git a/src/httpserver.h b/src/httpserver.h index 91ce5b4e00..f17be65962 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/init.cpp b/src/init.cpp index 539adc23d5..7215e87359 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1,52 +1,52 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "init.h" - -#include "addrman.h" -#include "amount.h" -#include "chain.h" -#include "chainparams.h" -#include "checkpoints.h" -#include "compat/sanity.h" -#include "consensus/validation.h" -#include "fs.h" -#include "httpserver.h" -#include "httprpc.h" -#include "key.h" -#include "validation.h" -#include "miner.h" -#include "netbase.h" -#include "net.h" -#include "net_processing.h" -#include "policy/feerate.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "rpc/server.h" -#include "rpc/register.h" -#include "rpc/safemode.h" -#include "rpc/blockchain.h" -#include "script/standard.h" -#include "script/sigcache.h" -#include "scheduler.h" -#include "timedata.h" -#include "txdb.h" -#include "txmempool.h" -#include "torcontrol.h" -#include "ui_interface.h" -#include "util.h" -#include "utilmoneystr.h" -#include "validationinterface.h" +#include <init.h> + +#include <addrman.h> +#include <amount.h> +#include <chain.h> +#include <chainparams.h> +#include <checkpoints.h> +#include <compat/sanity.h> +#include <consensus/validation.h> +#include <fs.h> +#include <httpserver.h> +#include <httprpc.h> +#include <key.h> +#include <validation.h> +#include <miner.h> +#include <netbase.h> +#include <net.h> +#include <net_processing.h> +#include <policy/feerate.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <rpc/server.h> +#include <rpc/register.h> +#include <rpc/safemode.h> +#include <rpc/blockchain.h> +#include <script/standard.h> +#include <script/sigcache.h> +#include <scheduler.h> +#include <timedata.h> +#include <txdb.h> +#include <txmempool.h> +#include <torcontrol.h> +#include <ui_interface.h> +#include <util.h> +#include <utilmoneystr.h> +#include <validationinterface.h> #ifdef ENABLE_WALLET -#include "wallet/init.h" +#include <wallet/init.h> #endif -#include "warnings.h" +#include <warnings.h> #include <stdint.h> #include <stdio.h> #include <memory> @@ -64,7 +64,7 @@ #include <openssl/crypto.h> #if ENABLE_ZMQ -#include "zmq/zmqnotificationinterface.h" +#include <zmq/zmqnotificationinterface.h> #endif bool fFeeEstimatesInitialized = false; @@ -152,7 +152,7 @@ public: // Writes do not need similar protection, as failure to write is handled by the caller. }; -static CCoinsViewErrorCatcher *pcoinscatcher = nullptr; +static std::unique_ptr<CCoinsViewErrorCatcher> pcoinscatcher; static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; void Interrupt(boost::thread_group& threadGroup) @@ -193,8 +193,8 @@ void Shutdown() // Because these depend on each-other, we make sure that neither can be // using the other before destroying them. - UnregisterValidationInterface(peerLogic.get()); - if(g_connman) g_connman->Stop(); + if (peerLogic) UnregisterValidationInterface(peerLogic.get()); + if (g_connman) g_connman->Stop(); peerLogic.reset(); g_connman.reset(); @@ -235,14 +235,10 @@ void Shutdown() if (pcoinsTip != nullptr) { FlushStateToDisk(); } - delete pcoinsTip; - pcoinsTip = nullptr; - delete pcoinscatcher; - pcoinscatcher = nullptr; - delete pcoinsdbview; - pcoinsdbview = nullptr; - delete pblocktree; - pblocktree = nullptr; + pcoinsTip.reset(); + pcoinscatcher.reset(); + pcoinsdbview.reset(); + pblocktree.reset(); } #ifdef ENABLE_WALLET StopWallets(); @@ -265,6 +261,7 @@ void Shutdown() #endif UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); + GetMainSignals().UnregisterWithMempoolSignals(mempool); #ifdef ENABLE_WALLET CloseWallets(); #endif @@ -345,6 +342,7 @@ std::string HelpMessage(HelpMessageMode mode) if (showDebug) strUsage += HelpMessageOpt("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER)); strUsage += HelpMessageOpt("-loadblock=<file>", _("Imports blocks from external blk000??.dat file on startup")); + strUsage += HelpMessageOpt("-debuglogfile=<file>", strprintf(_("Specify location of debug log file: this can be an absolute path or a path relative to the data directory (default: %s)"), DEFAULT_DEBUGLOGFILE)); strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=<n>", strprintf(_("Do not keep transactions in the mempool longer than <n> hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); @@ -369,11 +367,11 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); - strUsage += HelpMessageOpt("-addnode=<ip>", _("Add a node to connect to and attempt to keep the connection open")); + strUsage += HelpMessageOpt("-addnode=<ip>", _("Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info)")); strUsage += HelpMessageOpt("-banscore=<n>", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); strUsage += HelpMessageOpt("-bantime=<n>", strprintf(_("Number of seconds to keep misbehaving peers from reconnecting (default: %u)"), DEFAULT_MISBEHAVING_BANTIME)); strUsage += HelpMessageOpt("-bind=<addr>", _("Bind to given address and always listen on it. Use [host]:port notation for IPv6")); - strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s); -connect=0 disables automatic connections")); + strUsage += HelpMessageOpt("-connect=<ip>", _("Connect only to the specified node(s); -connect=0 disables automatic connections (the rules for this peer are the same as for -addnode)")); strUsage += HelpMessageOpt("-discover", _("Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)")); strUsage += HelpMessageOpt("-dns", _("Allow DNS lookups for -addnode, -seednode and -connect") + " " + strprintf(_("(default: %u)"), DEFAULT_NAME_LOOKUP)); strUsage += HelpMessageOpt("-dnsseed", _("Query for peer addresses via DNS lookup, if low on addresses (default: 1 unless -connect used)")); @@ -544,14 +542,14 @@ static void BlockNotifyCallback(bool initialSync, const CBlockIndex *pBlockIndex } static bool fHaveGenesis = false; -static boost::mutex cs_GenesisWait; +static CWaitableCriticalSection cs_GenesisWait; static CConditionVariable condvar_GenesisWait; static void BlockNotifyGenesisWait(bool, const CBlockIndex *pBlockIndex) { if (pBlockIndex != nullptr) { { - boost::unique_lock<boost::mutex> lock_GenesisWait(cs_GenesisWait); + WaitableLock lock_GenesisWait(cs_GenesisWait); fHaveGenesis = true; } condvar_GenesisWait.notify_all(); @@ -588,7 +586,7 @@ void CleanupBlockRevFiles() LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); fs::path blocksdir = GetDataDir() / "blocks"; for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { - if (is_regular_file(*it) && + if (fs::is_regular_file(*it) && it->path().filename().string().length() == 12 && it->path().filename().string().substr(8,4) == ".dat") { @@ -815,11 +813,10 @@ void InitLogging() namespace { // Variables internal to initialization process only -ServiceFlags nRelevantServices = NODE_NETWORK; int nMaxConnections; int nUserMaxConnections; int nFD; -ServiceFlags nLocalServices = NODE_NETWORK; +ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED); } // namespace @@ -923,7 +920,8 @@ bool AppInitParameterInteraction() // Special-case: if -debug=0/-nodebug is set, turn off debugging messages const std::vector<std::string> categories = gArgs.GetArgs("-debug"); - if (find(categories.begin(), categories.end(), std::string("0")) == categories.end()) { + if (std::none_of(categories.begin(), categories.end(), + [](std::string cat){return cat == "0" || cat == "none";})) { for (const auto& cat : categories) { uint32_t flag = 0; if (!GetLogCategory(&flag, &cat)) { @@ -1034,11 +1032,6 @@ bool AppInitParameterInteraction() fPruneMode = true; } - RegisterAllCoreRPCCommands(tableRPC); -#ifdef ENABLE_WALLET - RegisterWalletRPC(tableRPC); -#endif - nConnectTimeout = gArgs.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; @@ -1217,8 +1210,11 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) ShrinkDebugFile(); } - if (fPrintToDebugLog) - OpenDebugLog(); + if (fPrintToDebugLog) { + if (!OpenDebugLog()) { + return InitError(strprintf("Could not open debug log file %s", GetDebugLogPath().string())); + } + } if (!fLogTimestamps) LogPrintf("Startup time: %s\n", DateTimeStrFormat("%Y-%m-%d %H:%M:%S", GetTime())); @@ -1241,6 +1237,15 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) threadGroup.create_thread(boost::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + GetMainSignals().RegisterWithMempoolSignals(mempool); + + /* Register RPC commands regardless of -server setting so they will be + * available in the GUI RPC console even if external calls are disabled. + */ + RegisterAllCoreRPCCommands(tableRPC); +#ifdef ENABLE_WALLET + RegisterWalletRPC(tableRPC); +#endif /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections @@ -1271,7 +1276,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) g_connman = std::unique_ptr<CConnman>(new CConnman(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()))); CConnman& connman = *g_connman; - peerLogic.reset(new PeerLogicValidation(&connman)); + peerLogic.reset(new PeerLogicValidation(&connman, scheduler)); RegisterValidationInterface(peerLogic.get()); // sanitize comments per BIP-0014, format user agent and check total size @@ -1407,12 +1412,10 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) do { try { UnloadBlockIndex(); - delete pcoinsTip; - delete pcoinsdbview; - delete pcoinscatcher; - delete pblocktree; - - pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReset); + pcoinsTip.reset(); + pcoinsdbview.reset(); + pcoinscatcher.reset(); + pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, false, fReset)); if (fReset) { pblocktree->WriteReindexing(true); @@ -1463,8 +1466,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // At this point we're either in reindex or we've loaded a useful // block tree into mapBlockIndex! - pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReset || fReindexChainState); - pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); + pcoinsdbview.reset(new CCoinsViewDB(nCoinDBCache, false, fReset || fReindexChainState)); + pcoinscatcher.reset(new CCoinsViewErrorCatcher(pcoinsdbview.get())); // If necessary, upgrade from older database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate @@ -1474,13 +1477,13 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!ReplayBlocks(chainparams, pcoinsdbview)) { + if (!ReplayBlocks(chainparams, pcoinsdbview.get())) { strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate."); break; } // The on-disk coinsdb is now in a good state, create the cache - pcoinsTip = new CCoinsViewCache(pcoinscatcher); + pcoinsTip.reset(new CCoinsViewCache(pcoinscatcher.get())); bool is_coinsview_empty = fReset || fReindexChainState || pcoinsTip->GetBestBlock().IsNull(); if (!is_coinsview_empty) { @@ -1522,7 +1525,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (!CVerifyDB().VerifyDB(chainparams, pcoinsdbview, gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), + if (!CVerifyDB().VerifyDB(chainparams, pcoinsdbview.get(), gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { strLoadError = _("Corrupted block database detected"); break; @@ -1604,9 +1607,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // Note that setting NODE_WITNESS is never required: the only downside from not // doing so is that after activation, no upgraded nodes will fetch from you. nLocalServices = ServiceFlags(nLocalServices | NODE_WITNESS); - // Only care about others providing witness capabilities if there is a softfork - // defined. - nRelevantServices = ServiceFlags(nRelevantServices | NODE_WITNESS); } // ********************************************************* Step 10: import blocks @@ -1634,7 +1634,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // Wait for genesis block to be processed { - boost::unique_lock<boost::mutex> lock(cs_GenesisWait); + WaitableLock lock(cs_GenesisWait); while (!fHaveGenesis) { condvar_GenesisWait.wait(lock); } @@ -1663,7 +1663,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) CConnman::Options connOptions; connOptions.nLocalServices = nLocalServices; - connOptions.nRelevantServices = nRelevantServices; connOptions.nMaxConnections = nMaxConnections; connOptions.nMaxOutbound = std::min(MAX_OUTBOUND_CONNECTIONS, connOptions.nMaxConnections); connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; @@ -1727,5 +1726,5 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) StartWallets(scheduler); #endif - return !fRequestShutdown; + return true; } diff --git a/src/init.h b/src/init.h index a0a824738c..843024f02b 100644 --- a/src/init.h +++ b/src/init.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/key.cpp b/src/key.cpp index 315a3978c8..f8136f88b5 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "key.h" +#include <key.h> -#include "arith_uint256.h" -#include "crypto/common.h" -#include "crypto/hmac_sha512.h" -#include "pubkey.h" -#include "random.h" +#include <arith_uint256.h> +#include <crypto/common.h> +#include <crypto/hmac_sha512.h> +#include <random.h> #include <secp256k1.h> #include <secp256k1_recovery.h> @@ -16,43 +16,62 @@ static secp256k1_context* secp256k1_context_sign = nullptr; /** These functions are taken from the libsecp256k1 distribution and are very ugly. */ + +/** + * This parses a format loosely based on a DER encoding of the ECPrivateKey type from + * section C.4 of SEC 1 <http://www.secg.org/sec1-v2.pdf>, with the following caveats: + * + * * The octet-length of the SEQUENCE must be encoded as 1 or 2 octets. It is not + * required to be encoded as one octet if it is less than 256, as DER would require. + * * The octet-length of the SEQUENCE must not be greater than the remaining + * length of the key encoding, but need not match it (i.e. the encoding may contain + * junk after the encoded SEQUENCE). + * * The privateKey OCTET STRING is zero-filled on the left to 32 octets. + * * Anything after the encoding of the privateKey OCTET STRING is ignored, whether + * or not it is validly encoded DER. + * + * out32 must point to an output buffer of length at least 32 bytes. + */ static int ec_privkey_import_der(const secp256k1_context* ctx, unsigned char *out32, const unsigned char *privkey, size_t privkeylen) { const unsigned char *end = privkey + privkeylen; - int lenb = 0; - int len = 0; memset(out32, 0, 32); /* sequence header */ - if (end < privkey+1 || *privkey != 0x30) { + if (end - privkey < 1 || *privkey != 0x30u) { return 0; } privkey++; /* sequence length constructor */ - if (end < privkey+1 || !(*privkey & 0x80)) { + if (end - privkey < 1 || !(*privkey & 0x80u)) { return 0; } - lenb = *privkey & ~0x80; privkey++; + size_t lenb = *privkey & ~0x80u; privkey++; if (lenb < 1 || lenb > 2) { return 0; } - if (end < privkey+lenb) { + if (end - privkey < lenb) { return 0; } /* sequence length */ - len = privkey[lenb-1] | (lenb > 1 ? privkey[lenb-2] << 8 : 0); + size_t len = privkey[lenb-1] | (lenb > 1 ? privkey[lenb-2] << 8 : 0u); privkey += lenb; - if (end < privkey+len) { + if (end - privkey < len) { return 0; } /* sequence element 0: version number (=1) */ - if (end < privkey+3 || privkey[0] != 0x02 || privkey[1] != 0x01 || privkey[2] != 0x01) { + if (end - privkey < 3 || privkey[0] != 0x02u || privkey[1] != 0x01u || privkey[2] != 0x01u) { return 0; } privkey += 3; /* sequence element 1: octet string, up to 32 bytes */ - if (end < privkey+2 || privkey[0] != 0x04 || privkey[1] > 0x20 || end < privkey+2+privkey[1]) { + if (end - privkey < 2 || privkey[0] != 0x04u) { + return 0; + } + size_t oslen = privkey[1]; + privkey += 2; + if (oslen > 32 || end - privkey < oslen) { return 0; } - memcpy(out32 + 32 - privkey[1], privkey + 2, privkey[1]); + memcpy(out32 + (32 - oslen), privkey, oslen); if (!secp256k1_ec_seckey_verify(ctx, out32)) { memset(out32, 0, 32); return 0; @@ -60,7 +79,18 @@ static int ec_privkey_import_der(const secp256k1_context* ctx, unsigned char *ou return 1; } +/** + * This serializes to a DER encoding of the ECPrivateKey type from section C.4 of SEC 1 + * <http://www.secg.org/sec1-v2.pdf>. The optional parameters and publicKey fields are + * included. + * + * privkey must point to an output buffer of length at least CKey::PRIVATE_KEY_SIZE bytes. + * privkeylen must initially be set to the size of the privkey buffer. Upon return it + * will be set to the number of bytes used in the buffer. + * key32 must point to a 32-byte raw private key. + */ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *privkey, size_t *privkeylen, const unsigned char *key32, int compressed) { + assert(*privkeylen >= CKey::PRIVATE_KEY_SIZE); secp256k1_pubkey pubkey; size_t pubkeylen = 0; if (!secp256k1_ec_pubkey_create(ctx, &pubkey, key32)) { @@ -86,10 +116,11 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *pr memcpy(ptr, begin, sizeof(begin)); ptr += sizeof(begin); memcpy(ptr, key32, 32); ptr += 32; memcpy(ptr, middle, sizeof(middle)); ptr += sizeof(middle); - pubkeylen = 33; + pubkeylen = CPubKey::COMPRESSED_PUBLIC_KEY_SIZE; secp256k1_ec_pubkey_serialize(ctx, ptr, &pubkeylen, &pubkey, SECP256K1_EC_COMPRESSED); ptr += pubkeylen; *privkeylen = ptr - privkey; + assert(*privkeylen == CKey::COMPRESSED_PRIVATE_KEY_SIZE); } else { static const unsigned char begin[] = { 0x30,0x82,0x01,0x13,0x02,0x01,0x01,0x04,0x20 @@ -111,10 +142,11 @@ static int ec_privkey_export_der(const secp256k1_context *ctx, unsigned char *pr memcpy(ptr, begin, sizeof(begin)); ptr += sizeof(begin); memcpy(ptr, key32, 32); ptr += 32; memcpy(ptr, middle, sizeof(middle)); ptr += sizeof(middle); - pubkeylen = 65; + pubkeylen = CPubKey::PUBLIC_KEY_SIZE; secp256k1_ec_pubkey_serialize(ctx, ptr, &pubkeylen, &pubkey, SECP256K1_EC_UNCOMPRESSED); ptr += pubkeylen; *privkeylen = ptr - privkey; + assert(*privkeylen == CKey::PRIVATE_KEY_SIZE); } return 1; } @@ -136,8 +168,8 @@ CPrivKey CKey::GetPrivKey() const { CPrivKey privkey; int ret; size_t privkeylen; - privkey.resize(279); - privkeylen = 279; + privkey.resize(PRIVATE_KEY_SIZE); + privkeylen = PRIVATE_KEY_SIZE; ret = ec_privkey_export_der(secp256k1_context_sign, (unsigned char*) privkey.data(), &privkeylen, begin(), fCompressed ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); assert(ret); privkey.resize(privkeylen); @@ -147,7 +179,7 @@ CPrivKey CKey::GetPrivKey() const { CPubKey CKey::GetPubKey() const { assert(fValid); secp256k1_pubkey pubkey; - size_t clen = 65; + size_t clen = CPubKey::PUBLIC_KEY_SIZE; CPubKey result; int ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pubkey, begin()); assert(ret); @@ -160,8 +192,8 @@ CPubKey CKey::GetPubKey() const { bool CKey::Sign(const uint256 &hash, std::vector<unsigned char>& vchSig, uint32_t test_case) const { if (!fValid) return false; - vchSig.resize(72); - size_t nSigLen = 72; + vchSig.resize(CPubKey::SIGNATURE_SIZE); + size_t nSigLen = CPubKey::SIGNATURE_SIZE; unsigned char extra_entropy[32] = {0}; WriteLE32(extra_entropy, test_case); secp256k1_ecdsa_signature sig; @@ -189,7 +221,7 @@ bool CKey::VerifyPubKey(const CPubKey& pubkey) const { bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) const { if (!fValid) return false; - vchSig.resize(65); + vchSig.resize(CPubKey::COMPACT_SIGNATURE_SIZE); int rec = -1; secp256k1_ecdsa_recoverable_signature sig; int ret = secp256k1_ecdsa_sign_recoverable(secp256k1_context_sign, &sig, hash.begin(), begin(), secp256k1_nonce_function_rfc6979, nullptr); @@ -219,10 +251,10 @@ bool CKey::Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); if ((nChild >> 31) == 0) { CPubKey pubkey = GetPubKey(); - assert(pubkey.begin() + 33 == pubkey.end()); + assert(pubkey.size() == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); BIP32Hash(cc, nChild, *pubkey.begin(), pubkey.begin()+1, vout.data()); } else { - assert(begin() + 32 == end()); + assert(size() == 32); BIP32Hash(cc, nChild, 0, begin(), vout.data()); } memcpy(ccChild.begin(), vout.data()+32, 32); @@ -1,39 +1,45 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_KEY_H #define BITCOIN_KEY_H -#include "pubkey.h" -#include "serialize.h" -#include "support/allocators/secure.h" -#include "uint256.h" +#include <pubkey.h> +#include <serialize.h> +#include <support/allocators/secure.h> +#include <uint256.h> #include <stdexcept> #include <vector> /** - * secp256k1: - * const unsigned int PRIVATE_KEY_SIZE = 279; - * const unsigned int PUBLIC_KEY_SIZE = 65; - * const unsigned int SIGNATURE_SIZE = 72; - * - * see www.keylength.com - * script supports up to 75 for single byte push - */ - -/** * secure_allocator is defined in allocators.h - * CPrivKey is a serialized private key, with all parameters included (279 bytes) + * CPrivKey is a serialized private key, with all parameters included + * (PRIVATE_KEY_SIZE bytes) */ typedef std::vector<unsigned char, secure_allocator<unsigned char> > CPrivKey; /** An encapsulated private key. */ class CKey { +public: + /** + * secp256k1: + */ + static const unsigned int PRIVATE_KEY_SIZE = 279; + static const unsigned int COMPRESSED_PRIVATE_KEY_SIZE = 214; + /** + * see www.keylength.com + * script supports up to 75 for single byte push + */ + static_assert( + PRIVATE_KEY_SIZE >= COMPRESSED_PRIVATE_KEY_SIZE, + "COMPRESSED_PRIVATE_KEY_SIZE is larger than PRIVATE_KEY_SIZE"); + private: //! Whether this private key is valid. We check for correctness when modifying the key //! data, so fValid should always correspond to the actual state. diff --git a/src/keystore.cpp b/src/keystore.cpp index 8454175ca8..fab1b81c9a 100644 --- a/src/keystore.cpp +++ b/src/keystore.cpp @@ -1,18 +1,41 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "keystore.h" +#include <keystore.h> -#include "key.h" -#include "pubkey.h" -#include "util.h" +#include <util.h> bool CKeyStore::AddKey(const CKey &key) { return AddKeyPubKey(key, key.GetPubKey()); } +void CBasicKeyStore::ImplicitlyLearnRelatedKeyScripts(const CPubKey& pubkey) +{ + AssertLockHeld(cs_KeyStore); + CKeyID key_id = pubkey.GetID(); + // We must actually know about this key already. + assert(HaveKey(key_id) || mapWatchKeys.count(key_id)); + // This adds the redeemscripts necessary to detect P2WPKH and P2SH-P2WPKH + // outputs. Technically P2WPKH outputs don't have a redeemscript to be + // spent. However, our current IsMine logic requires the corresponding + // P2SH-P2WPKH redeemscript to be present in the wallet in order to accept + // payment even to P2WPKH outputs. + // Also note that having superfluous scripts in the keystore never hurts. + // They're only used to guide recursion in signing and IsMine logic - if + // a script is present but we can't do anything with it, it has no effect. + // "Implicitly" refers to fact that scripts are derived automatically from + // existing keys, and are present in memory, even without being explicitly + // loaded (e.g. from a file). + if (pubkey.IsCompressed()) { + CScript script = GetScriptForDestination(WitnessV0KeyHash(key_id)); + // This does not use AddCScript, as it may be overridden. + CScriptID id(script); + mapScripts[id] = std::move(script); + } +} + bool CBasicKeyStore::GetPubKey(const CKeyID &address, CPubKey &vchPubKeyOut) const { CKey key; @@ -33,9 +56,37 @@ bool CBasicKeyStore::AddKeyPubKey(const CKey& key, const CPubKey &pubkey) { LOCK(cs_KeyStore); mapKeys[pubkey.GetID()] = key; + ImplicitlyLearnRelatedKeyScripts(pubkey); return true; } +bool CBasicKeyStore::HaveKey(const CKeyID &address) const +{ + LOCK(cs_KeyStore); + return mapKeys.count(address) > 0; +} + +std::set<CKeyID> CBasicKeyStore::GetKeys() const +{ + LOCK(cs_KeyStore); + std::set<CKeyID> set_address; + for (const auto& mi : mapKeys) { + set_address.insert(mi.first); + } + return set_address; +} + +bool CBasicKeyStore::GetKey(const CKeyID &address, CKey &keyOut) const +{ + LOCK(cs_KeyStore); + KeyMap::const_iterator mi = mapKeys.find(address); + if (mi != mapKeys.end()) { + keyOut = mi->second; + return true; + } + return false; +} + bool CBasicKeyStore::AddCScript(const CScript& redeemScript) { if (redeemScript.size() > MAX_SCRIPT_ELEMENT_SIZE) @@ -52,6 +103,16 @@ bool CBasicKeyStore::HaveCScript(const CScriptID& hash) const return mapScripts.count(hash) > 0; } +std::set<CScriptID> CBasicKeyStore::GetCScripts() const +{ + LOCK(cs_KeyStore); + std::set<CScriptID> set_script; + for (const auto& mi : mapScripts) { + set_script.insert(mi.first); + } + return set_script; +} + bool CBasicKeyStore::GetCScript(const CScriptID &hash, CScript& redeemScriptOut) const { LOCK(cs_KeyStore); @@ -85,8 +146,10 @@ bool CBasicKeyStore::AddWatchOnly(const CScript &dest) LOCK(cs_KeyStore); setWatchOnly.insert(dest); CPubKey pubKey; - if (ExtractPubKey(dest, pubKey)) + if (ExtractPubKey(dest, pubKey)) { mapWatchKeys[pubKey.GetID()] = pubKey; + ImplicitlyLearnRelatedKeyScripts(pubKey); + } return true; } @@ -95,8 +158,11 @@ bool CBasicKeyStore::RemoveWatchOnly(const CScript &dest) LOCK(cs_KeyStore); setWatchOnly.erase(dest); CPubKey pubKey; - if (ExtractPubKey(dest, pubKey)) + if (ExtractPubKey(dest, pubKey)) { mapWatchKeys.erase(pubKey.GetID()); + } + // Related CScripts are not removed; having superfluous scripts around is + // harmless (see comment in ImplicitlyLearnRelatedKeyScripts). return true; } @@ -111,3 +177,25 @@ bool CBasicKeyStore::HaveWatchOnly() const LOCK(cs_KeyStore); return (!setWatchOnly.empty()); } + +CKeyID GetKeyForDestination(const CKeyStore& store, const CTxDestination& dest) +{ + // Only supports destinations which map to single public keys, i.e. P2PKH, + // P2WPKH, and P2SH-P2WPKH. + if (auto id = boost::get<CKeyID>(&dest)) { + return *id; + } + if (auto witness_id = boost::get<WitnessV0KeyHash>(&dest)) { + return CKeyID(*witness_id); + } + if (auto script_id = boost::get<CScriptID>(&dest)) { + CScript script; + CTxDestination inner_dest; + if (store.GetCScript(*script_id, script) && ExtractDestination(script, inner_dest)) { + if (auto inner_witness_id = boost::get<WitnessV0KeyHash>(&inner_dest)) { + return CKeyID(*inner_witness_id); + } + } + } + return CKeyID(); +} diff --git a/src/keystore.h b/src/keystore.h index 9b85ddb0ec..ffd3238fd6 100644 --- a/src/keystore.h +++ b/src/keystore.h @@ -1,16 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_KEYSTORE_H #define BITCOIN_KEYSTORE_H -#include "key.h" -#include "pubkey.h" -#include "script/script.h" -#include "script/standard.h" -#include "sync.h" +#include <key.h> +#include <pubkey.h> +#include <script/script.h> +#include <script/standard.h> +#include <sync.h> #include <boost/signals2/signal.hpp> @@ -36,6 +36,7 @@ public: //! Support for BIP 0013 : see https://github.com/bitcoin/bips/blob/master/bip-0013.mediawiki virtual bool AddCScript(const CScript& redeemScript) =0; virtual bool HaveCScript(const CScriptID &hash) const =0; + virtual std::set<CScriptID> GetCScripts() const =0; virtual bool GetCScript(const CScriptID &hash, CScript& redeemScriptOut) const =0; //! Support for Watch-only addresses @@ -59,42 +60,17 @@ protected: ScriptMap mapScripts; WatchOnlySet setWatchOnly; + void ImplicitlyLearnRelatedKeyScripts(const CPubKey& pubkey); + public: bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override; bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; - bool HaveKey(const CKeyID &address) const override - { - bool result; - { - LOCK(cs_KeyStore); - result = (mapKeys.count(address) > 0); - } - return result; - } - std::set<CKeyID> GetKeys() const override - { - LOCK(cs_KeyStore); - std::set<CKeyID> set_address; - for (const auto& mi : mapKeys) { - set_address.insert(mi.first); - } - return set_address; - } - bool GetKey(const CKeyID &address, CKey &keyOut) const override - { - { - LOCK(cs_KeyStore); - KeyMap::const_iterator mi = mapKeys.find(address); - if (mi != mapKeys.end()) - { - keyOut = mi->second; - return true; - } - } - return false; - } + bool HaveKey(const CKeyID &address) const override; + std::set<CKeyID> GetKeys() const override; + bool GetKey(const CKeyID &address, CKey &keyOut) const override; bool AddCScript(const CScript& redeemScript) override; bool HaveCScript(const CScriptID &hash) const override; + std::set<CScriptID> GetCScripts() const override; bool GetCScript(const CScriptID &hash, CScript& redeemScriptOut) const override; bool AddWatchOnly(const CScript &dest) override; @@ -106,4 +82,7 @@ public: typedef std::vector<unsigned char, secure_allocator<unsigned char> > CKeyingMaterial; typedef std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char> > > CryptedKeyMap; +/** Return the CKeyID of the key involved in a script (if there is a unique one). */ +CKeyID GetKeyForDestination(const CKeyStore& store, const CTxDestination& dest); + #endif // BITCOIN_KEYSTORE_H diff --git a/src/limitedmap.h b/src/limitedmap.h index 7afc8b458d..d5f00f1fee 100644 --- a/src/limitedmap.h +++ b/src/limitedmap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/memusage.h b/src/memusage.h index 93fd6a0eb5..fea7ecdf9f 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -1,11 +1,11 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_MEMUSAGE_H #define BITCOIN_MEMUSAGE_H -#include "indirectmap.h" +#include <indirectmap.h> #include <stdlib.h> diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index 3f07b4dac4..de3519c1da 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "merkleblock.h" +#include <merkleblock.h> -#include "hash.h" -#include "consensus/consensus.h" -#include "utilstrencodings.h" +#include <hash.h> +#include <consensus/consensus.h> +#include <utilstrencodings.h> CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std::set<uint256>* txids) diff --git a/src/merkleblock.h b/src/merkleblock.h index 6c05f2c1f8..0976e21c3a 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -1,15 +1,15 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_MERKLEBLOCK_H #define BITCOIN_MERKLEBLOCK_H -#include "serialize.h" -#include "uint256.h" -#include "primitives/block.h" -#include "bloom.h" +#include <serialize.h> +#include <uint256.h> +#include <primitives/block.h> +#include <bloom.h> #include <vector> diff --git a/src/miner.cpp b/src/miner.cpp index a9989f4b17..4e63ab4df0 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -1,31 +1,30 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "miner.h" - -#include "amount.h" -#include "chain.h" -#include "chainparams.h" -#include "coins.h" -#include "consensus/consensus.h" -#include "consensus/tx_verify.h" -#include "consensus/merkle.h" -#include "consensus/validation.h" -#include "hash.h" -#include "validation.h" -#include "net.h" -#include "policy/feerate.h" -#include "policy/policy.h" -#include "pow.h" -#include "primitives/transaction.h" -#include "script/standard.h" -#include "timedata.h" -#include "txmempool.h" -#include "util.h" -#include "utilmoneystr.h" -#include "validationinterface.h" +#include <miner.h> + +#include <amount.h> +#include <chain.h> +#include <chainparams.h> +#include <coins.h> +#include <consensus/consensus.h> +#include <consensus/tx_verify.h> +#include <consensus/merkle.h> +#include <consensus/validation.h> +#include <hash.h> +#include <validation.h> +#include <net.h> +#include <policy/feerate.h> +#include <policy/policy.h> +#include <pow.h> +#include <primitives/transaction.h> +#include <script/standard.h> +#include <timedata.h> +#include <util.h> +#include <utilmoneystr.h> +#include <validationinterface.h> #include <algorithm> #include <queue> diff --git a/src/miner.h b/src/miner.h index db165e71c6..698b4a4788 100644 --- a/src/miner.h +++ b/src/miner.h @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_MINER_H #define BITCOIN_MINER_H -#include "primitives/block.h" -#include "txmempool.h" +#include <primitives/block.h> +#include <txmempool.h> #include <stdint.h> #include <memory> @@ -71,7 +71,7 @@ struct modifiedentry_iter { // except operating on CTxMemPoolModifiedEntry. // TODO: refactor to avoid duplication of this logic. struct CompareModifiedEntry { - bool operator()(const CTxMemPoolModifiedEntry &a, const CTxMemPoolModifiedEntry &b) + bool operator()(const CTxMemPoolModifiedEntry &a, const CTxMemPoolModifiedEntry &b) const { double f1 = (double)a.nModFeesWithAncestors * b.nSizeWithAncestors; double f2 = (double)b.nModFeesWithAncestors * a.nSizeWithAncestors; @@ -86,7 +86,7 @@ struct CompareModifiedEntry { // This is sufficient to sort an ancestor package in an order that is valid // to appear in a block. struct CompareTxIterByAncestorCount { - bool operator()(const CTxMemPool::txiter &a, const CTxMemPool::txiter &b) + bool operator()(const CTxMemPool::txiter &a, const CTxMemPool::txiter &b) const { if (a->GetCountWithAncestors() != b->GetCountWithAncestors()) return a->GetCountWithAncestors() < b->GetCountWithAncestors(); diff --git a/src/net.cpp b/src/net.cpp index ea3840a708..682f47bff2 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1,26 +1,24 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "net.h" - -#include "addrman.h" -#include "chainparams.h" -#include "clientversion.h" -#include "consensus/consensus.h" -#include "crypto/common.h" -#include "crypto/sha256.h" -#include "hash.h" -#include "primitives/transaction.h" -#include "netbase.h" -#include "scheduler.h" -#include "ui_interface.h" -#include "utilstrencodings.h" +#include <net.h> + +#include <chainparams.h> +#include <clientversion.h> +#include <consensus/consensus.h> +#include <crypto/common.h> +#include <crypto/sha256.h> +#include <primitives/transaction.h> +#include <netbase.h> +#include <scheduler.h> +#include <ui_interface.h> +#include <utilstrencodings.h> #ifdef WIN32 #include <string.h> @@ -110,13 +108,13 @@ bool GetLocal(CService& addr, const CNetAddr *paddrPeer) int nBestReachability = -1; { LOCK(cs_mapLocalHost); - for (std::map<CNetAddr, LocalServiceInfo>::iterator it = mapLocalHost.begin(); it != mapLocalHost.end(); it++) + for (const auto& entry : mapLocalHost) { - int nScore = (*it).second.nScore; - int nReachability = (*it).first.GetReachabilityFrom(paddrPeer); + int nScore = entry.second.nScore; + int nReachability = entry.first.GetReachabilityFrom(paddrPeer); if (nReachability > nBestReachability || (nReachability == nBestReachability && nScore > nBestScore)) { - addr = CService((*it).first, (*it).second.nPort); + addr = CService(entry.first, entry.second.nPort); nBestReachability = nReachability; nBestScore = nScore; } @@ -417,40 +415,48 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo if (addrConnect.IsValid()) { bool proxyConnectionFailed = false; - if (GetProxy(addrConnect.GetNetwork(), proxy)) + if (GetProxy(addrConnect.GetNetwork(), proxy)) { + hSocket = CreateSocket(proxy.proxy); + if (hSocket == INVALID_SOCKET) { + return nullptr; + } connected = ConnectThroughProxy(proxy, addrConnect.ToStringIP(), addrConnect.GetPort(), hSocket, nConnectTimeout, &proxyConnectionFailed); - else // no proxy needed (none set for target network) + } else { + // no proxy needed (none set for target network) + hSocket = CreateSocket(addrConnect); + if (hSocket == INVALID_SOCKET) { + return nullptr; + } connected = ConnectSocketDirectly(addrConnect, hSocket, nConnectTimeout); + } if (!proxyConnectionFailed) { // If a connection to the node was attempted, and failure (if any) is not caused by a problem connecting to // the proxy, mark this as an attempt. addrman.Attempt(addrConnect, fCountFailure); } } else if (pszDest && GetNameProxy(proxy)) { + hSocket = CreateSocket(proxy.proxy); + if (hSocket == INVALID_SOCKET) { + return nullptr; + } std::string host; int port = default_port; SplitHostPort(std::string(pszDest), port, host); connected = ConnectThroughProxy(proxy, host, port, hSocket, nConnectTimeout, nullptr); } - if (connected) { - if (!IsSelectableSocket(hSocket)) { - LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); - CloseSocket(hSocket); - return nullptr; - } - - // Add node - NodeId id = GetNewNodeId(); - uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); - CAddress addr_bind = GetBindAddress(hSocket); - CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false); - pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); - pnode->AddRef(); - - return pnode; + if (!connected) { + CloseSocket(hSocket); + return nullptr; } - return nullptr; + // Add node + NodeId id = GetNewNodeId(); + uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + CAddress addr_bind = GetBindAddress(hSocket); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false); + pnode->AddRef(); + + return pnode; } void CConnman::DumpBanlist() @@ -597,21 +603,28 @@ void CConnman::SetBanned(const banmap_t &banMap) void CConnman::SweepBanned() { int64_t now = GetTime(); - - LOCK(cs_setBanned); - banmap_t::iterator it = setBanned.begin(); - while(it != setBanned.end()) + bool notifyUI = false; { - CSubNet subNet = (*it).first; - CBanEntry banEntry = (*it).second; - if(now > banEntry.nBanUntil) + LOCK(cs_setBanned); + banmap_t::iterator it = setBanned.begin(); + while(it != setBanned.end()) { - setBanned.erase(it++); - setBannedIsDirty = true; - LogPrint(BCLog::NET, "%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, subNet.ToString()); + CSubNet subNet = (*it).first; + CBanEntry banEntry = (*it).second; + if(now > banEntry.nBanUntil) + { + setBanned.erase(it++); + setBannedIsDirty = true; + notifyUI = true; + LogPrint(BCLog::NET, "%s: Removed banned node ip/subnet from banlist.dat: %s\n", __func__, subNet.ToString()); + } + else + ++it; } - else - ++it; + } + // update UI + if(notifyUI && clientInterface) { + clientInterface->BannedListChanged(); } } @@ -685,7 +698,7 @@ void CNode::copyStats(CNodeStats &stats) X(cleanSubVer); } X(fInbound); - X(fAddnode); + X(m_manual_connection); X(nStartingHeight); { LOCK(cs_vSend); @@ -962,6 +975,16 @@ static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEviction return a.nTimeConnected > b.nTimeConnected; } + +//! Sort an array by the specified comparator, then erase the last K elements. +template<typename T, typename Comparator> +static void EraseLastKElements(std::vector<T> &elements, Comparator comparator, size_t k) +{ + std::sort(elements.begin(), elements.end(), comparator); + size_t eraseSize = std::min(k, elements.size()); + elements.erase(elements.end() - eraseSize, elements.end()); +} + /** Try to find a connection to evict when the node is full. * Extreme care must be taken to avoid opening the node to attacker * triggered network partitioning. @@ -985,48 +1008,29 @@ bool CConnman::AttemptToEvictConnection() continue; NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, node->nLastBlockTime, node->nLastTXTime, - (node->nServices & nRelevantServices) == nRelevantServices, + HasAllDesirableServiceFlags(node->nServices), node->fRelayTxes, node->pfilter != nullptr, node->addr, node->nKeyedNetGroup}; vEvictionCandidates.push_back(candidate); } } - if (vEvictionCandidates.empty()) return false; - // Protect connections with certain characteristics // Deterministically select 4 peers to protect by netgroup. // An attacker cannot predict which netgroups will be protected - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNetGroupKeyed); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) return false; - + EraseLastKElements(vEvictionCandidates, CompareNetGroupKeyed, 4); // Protect the 8 nodes with the lowest minimum ping time. // An attacker cannot manipulate this metric without physically moving nodes closer to the target. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeMinPingTime); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(8, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) return false; - + EraseLastKElements(vEvictionCandidates, ReverseCompareNodeMinPingTime, 8); // Protect 4 nodes that most recently sent us transactions. // An attacker cannot manipulate this metric without performing useful work. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNodeTXTime); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) return false; - + EraseLastKElements(vEvictionCandidates, CompareNodeTXTime, 4); // Protect 4 nodes that most recently sent us blocks. // An attacker cannot manipulate this metric without performing useful work. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNodeBlockTime); - vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); - - if (vEvictionCandidates.empty()) return false; - + EraseLastKElements(vEvictionCandidates, CompareNodeBlockTime, 4); // Protect the half of the remaining nodes which have been connected the longest. // This replicates the non-eviction implicit behavior, and precludes attacks that start later. - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeTimeConnected); - vEvictionCandidates.erase(vEvictionCandidates.end() - static_cast<int>(vEvictionCandidates.size() / 2), vEvictionCandidates.end()); + EraseLastKElements(vEvictionCandidates, ReverseCompareNodeTimeConnected, vEvictionCandidates.size() / 2); if (vEvictionCandidates.empty()) return false; @@ -1037,12 +1041,12 @@ bool CConnman::AttemptToEvictConnection() int64_t nMostConnectionsTime = 0; std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapNetGroupNodes; for (const NodeEvictionCandidate &node : vEvictionCandidates) { - mapNetGroupNodes[node.nKeyedNetGroup].push_back(node); - int64_t grouptime = mapNetGroupNodes[node.nKeyedNetGroup][0].nTimeConnected; - size_t groupsize = mapNetGroupNodes[node.nKeyedNetGroup].size(); + std::vector<NodeEvictionCandidate> &group = mapNetGroupNodes[node.nKeyedNetGroup]; + group.push_back(node); + int64_t grouptime = group[0].nTimeConnected; - if (groupsize > nMostConnections || (groupsize == nMostConnections && grouptime > nMostConnectionsTime)) { - nMostConnections = groupsize; + if (group.size() > nMostConnections || (group.size() == nMostConnections && grouptime > nMostConnectionsTime)) { + nMostConnections = group.size(); nMostConnectionsTime = grouptime; naMostConnections = node.nKeyedNetGroup; } @@ -1112,7 +1116,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { if (IsBanned(addr) && !whitelisted) { - LogPrintf("connection from %s dropped (banned)\n", addr.ToString()); + LogPrint(BCLog::NET, "connection from %s dropped (banned)\n", addr.ToString()); CloseSocket(hSocket); return; } @@ -1544,22 +1548,20 @@ void ThreadMapPort() void MapPort(bool fUseUPnP) { - static boost::thread* upnp_thread = nullptr; + static std::unique_ptr<boost::thread> upnp_thread; if (fUseUPnP) { if (upnp_thread) { upnp_thread->interrupt(); upnp_thread->join(); - delete upnp_thread; } - upnp_thread = new boost::thread(boost::bind(&TraceThread<void (*)()>, "upnp", &ThreadMapPort)); + upnp_thread.reset(new boost::thread(boost::bind(&TraceThread<void (*)()>, "upnp", &ThreadMapPort))); } else if (upnp_thread) { upnp_thread->interrupt(); upnp_thread->join(); - delete upnp_thread; - upnp_thread = nullptr; + upnp_thread.reset(); } } @@ -1602,7 +1604,7 @@ void CConnman::ThreadDNSAddressSeed() LOCK(cs_vNodes); int nRelevant = 0; for (auto pnode : vNodes) { - nRelevant += pnode->fSuccessfullyConnected && ((pnode->nServices & nRelevantServices) == nRelevantServices); + nRelevant += pnode->fSuccessfullyConnected && !pnode->fFeeler && !pnode->fOneShot && !pnode->m_manual_connection && !pnode->fInbound; } if (nRelevant >= 2) { LogPrintf("P2P peers available. Skipped DNS seeding.\n"); @@ -1624,7 +1626,7 @@ void CConnman::ThreadDNSAddressSeed() } else { std::vector<CNetAddr> vIPs; std::vector<CAddress> vAdd; - ServiceFlags requiredServiceBits = nRelevantServices; + ServiceFlags requiredServiceBits = GetDesirableServiceFlags(NODE_NONE); std::string host = GetDNSHost(seed, &requiredServiceBits); CNetAddr resolveSource; if (!resolveSource.SetInternal(host)) { @@ -1694,6 +1696,37 @@ void CConnman::ProcessOneShot() } } +bool CConnman::GetTryNewOutboundPeer() +{ + return m_try_another_outbound_peer; +} + +void CConnman::SetTryNewOutboundPeer(bool flag) +{ + m_try_another_outbound_peer = flag; + LogPrint(BCLog::NET, "net: setting try another outbound peer=%s\n", flag ? "true" : "false"); +} + +// Return the number of peers we have over our outbound connection limit +// Exclude peers that are marked for disconnect, or are going to be +// disconnected soon (eg one-shots and feelers) +// Also exclude peers that haven't finished initial connection handshake yet +// (so that we don't decide we're over our desired connection limit, and then +// evict some peer that has finished the handshake) +int CConnman::GetExtraOutboundCount() +{ + int nOutbound = 0; + { + LOCK(cs_vNodes); + for (CNode* pnode : vNodes) { + if (!pnode->fInbound && !pnode->m_manual_connection && !pnode->fFeeler && !pnode->fDisconnect && !pnode->fOneShot && pnode->fSuccessfullyConnected) { + ++nOutbound; + } + } + } + return std::max(nOutbound - nMaxOutbound, 0); +} + void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { // Connect to specific addresses @@ -1705,7 +1738,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) for (const std::string& strAddr : connect) { CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, nullptr, strAddr.c_str()); + OpenNetworkConnection(addr, false, nullptr, strAddr.c_str(), false, false, true); for (int i = 0; i < 10 && i < nLoop; i++) { if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) @@ -1753,17 +1786,11 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Only connect out to one peer per network group (/16 for IPv4). // Do this here so we don't have to critsect vNodes inside mapAddresses critsect. int nOutbound = 0; - int nOutboundRelevant = 0; std::set<std::vector<unsigned char> > setConnected; { LOCK(cs_vNodes); for (CNode* pnode : vNodes) { - if (!pnode->fInbound && !pnode->fAddnode) { - - // Count the peers that have all relevant services - if (pnode->fSuccessfullyConnected && !pnode->fFeeler && ((pnode->nServices & nRelevantServices) == nRelevantServices)) { - nOutboundRelevant++; - } + if (!pnode->fInbound && !pnode->m_manual_connection) { // Netgroups for inbound and addnode peers are not excluded because our goal here // is to not use multiple of our limited outbound slots on a single netgroup // but inbound and addnode peers do not use our outbound slots. Inbound peers @@ -1788,7 +1815,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // * Only make a feeler connection once every few minutes. // bool fFeeler = false; - if (nOutbound >= nMaxOutbound) { + + if (nOutbound >= nMaxOutbound && !GetTryNewOutboundPeer()) { int64_t nTime = GetTimeMicros(); // The current time right now (in microseconds). if (nTime > nNextFeeler) { nNextFeeler = PoissonNextSend(nTime, FEELER_INTERVAL); @@ -1818,21 +1846,16 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) if (IsLimited(addr)) continue; - // only connect to full nodes - if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) - continue; - // only consider very recently tried nodes after 30 failed attempts if (nANow - addr.nLastTry < 600 && nTries < 30) continue; - // only consider nodes missing relevant services after 40 failed attempts and only if less than half the outbound are up. - ServiceFlags nRequiredServices = nRelevantServices; - if (nTries >= 40 && nOutbound < (nMaxOutbound >> 1)) { - nRequiredServices = REQUIRED_SERVICES; - } - - if ((addr.nServices & nRequiredServices) != nRequiredServices) { + // for non-feelers, require all the services we'll want, + // for feelers, only require they be a full node (only because most + // SPV clients don't have a good address DB available) + if (!fFeeler && !HasAllDesirableServiceFlags(addr.nServices)) { + continue; + } else if (fFeeler && !MayHaveUsefulAddressDB(addr.nServices)) { continue; } @@ -1841,13 +1864,6 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) continue; addrConnect = addr; - - // regardless of the services assumed to be available, only require the minimum if half or more outbound have relevant services - if (nOutboundRelevant >= (nMaxOutbound >> 1)) { - addrConnect.nServices = REQUIRED_SERVICES; - } else { - addrConnect.nServices = nRequiredServices; - } break; } @@ -1946,7 +1962,7 @@ void CConnman::ThreadOpenAddedConnections() } // if successful, this moves the passed grant to the constructed node -bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool fAddnode) +bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound, const char *pszDest, bool fOneShot, bool fFeeler, bool manual_connection) { // // Initiate outbound network connection @@ -1975,8 +1991,8 @@ bool CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai pnode->fOneShot = true; if (fFeeler) pnode->fFeeler = true; - if (fAddnode) - pnode->fAddnode = true; + if (manual_connection) + pnode->m_manual_connection = true; m_msgproc->InitializeNode(pnode); { @@ -2056,44 +2072,21 @@ bool CConnman::BindListenPort(const CService &addrBind, std::string& strError, b return false; } - SOCKET hListenSocket = socket(((struct sockaddr*)&sockaddr)->sa_family, SOCK_STREAM, IPPROTO_TCP); + SOCKET hListenSocket = CreateSocket(addrBind); if (hListenSocket == INVALID_SOCKET) { strError = strprintf("Error: Couldn't open socket for incoming connections (socket returned error %s)", NetworkErrorString(WSAGetLastError())); LogPrintf("%s\n", strError); return false; } - if (!IsSelectableSocket(hListenSocket)) - { - strError = "Error: Couldn't create a listenable socket for incoming connections"; - LogPrintf("%s\n", strError); - return false; - } - - #ifndef WIN32 -#ifdef SO_NOSIGPIPE - // Different way of disabling SIGPIPE on BSD - setsockopt(hListenSocket, SOL_SOCKET, SO_NOSIGPIPE, (void*)&nOne, sizeof(int)); -#endif // Allow binding if the port is still in TIME_WAIT state after // the program was closed and restarted. setsockopt(hListenSocket, SOL_SOCKET, SO_REUSEADDR, (void*)&nOne, sizeof(int)); - // Disable Nagle's algorithm - setsockopt(hListenSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&nOne, sizeof(int)); #else setsockopt(hListenSocket, SOL_SOCKET, SO_REUSEADDR, (const char*)&nOne, sizeof(int)); - setsockopt(hListenSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&nOne, sizeof(int)); #endif - // Set to non-blocking, incoming connections will also inherit this - if (!SetSocketNonBlocking(hListenSocket, true)) { - CloseSocket(hListenSocket); - strError = strprintf("BindListenPort: Setting listening socket to non-blocking failed, error %s\n", NetworkErrorString(WSAGetLastError())); - LogPrintf("%s\n", strError); - return false; - } - // some systems don't have IPV6_V6ONLY but are always v6only; others do have the option // and enable it by default or not. Try to enable it, if possible. if (addrBind.IsIPv6()) { @@ -2220,9 +2213,8 @@ CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In) : nSeed0(nSeed0In), nSe nLastNodeId = 0; nSendBufferMaxSize = 0; nReceiveFloodSize = 0; - semOutbound = nullptr; - semAddnode = nullptr; flagInterruptMsgProc = false; + SetTryNewOutboundPeer(false); Options connOptions; Init(connOptions); @@ -2268,10 +2260,16 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) { Init(connOptions); - nTotalBytesRecv = 0; - nTotalBytesSent = 0; - nMaxOutboundTotalBytesSentInCycle = 0; - nMaxOutboundCycleStartTime = 0; + { + LOCK(cs_totalBytesRecv); + nTotalBytesRecv = 0; + } + { + LOCK(cs_totalBytesSent); + nTotalBytesSent = 0; + nMaxOutboundTotalBytesSentInCycle = 0; + nMaxOutboundCycleStartTime = 0; + } if (fListen && !InitBinds(connOptions.vBinds, connOptions.vWhiteBinds)) { if (clientInterface) { @@ -2326,11 +2324,11 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) if (semOutbound == nullptr) { // initialize semaphore - semOutbound = new CSemaphore(std::min((nMaxOutbound + nMaxFeeler), nMaxConnections)); + semOutbound = MakeUnique<CSemaphore>(std::min((nMaxOutbound + nMaxFeeler), nMaxConnections)); } if (semAddnode == nullptr) { // initialize semaphore - semAddnode = new CSemaphore(nMaxAddnode); + semAddnode = MakeUnique<CSemaphore>(nMaxAddnode); } // @@ -2453,10 +2451,8 @@ void CConnman::Stop() vNodes.clear(); vNodesDisconnected.clear(); vhListenSocket.clear(); - delete semOutbound; - semOutbound = nullptr; - delete semAddnode; - semAddnode = nullptr; + semOutbound.reset(); + semAddnode.reset(); } void CConnman::DeleteNode(CNode* pnode) @@ -2712,7 +2708,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn nSendVersion(0) { nServices = NODE_NONE; - nServicesExpected = NODE_NONE; hSocket = hSocketIn; nRecvVersion = INIT_PROTO_VERSION; nLastSend = 0; @@ -2725,7 +2720,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn strSubVer = ""; fWhitelisted = false; fOneShot = false; - fAddnode = false; + m_manual_connection = false; fClient = false; // set by version message fFeeler = false; fSuccessfullyConnected = false; @@ -2743,7 +2738,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn nNextInvSend = 0; fRelayTxes = false; fSentAddr = false; - pfilter = new CBloomFilter(); + pfilter = MakeUnique<CBloomFilter>(); timeLastMempoolReq = 0; nLastBlockTime = 0; nLastTXTime = 0; @@ -2773,9 +2768,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn CNode::~CNode() { CloseSocket(hSocket); - - if (pfilter) - delete pfilter; } void CNode::AskFor(const CInv& inv) @@ -1,26 +1,26 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_NET_H #define BITCOIN_NET_H -#include "addrdb.h" -#include "addrman.h" -#include "amount.h" -#include "bloom.h" -#include "compat.h" -#include "hash.h" -#include "limitedmap.h" -#include "netaddress.h" -#include "policy/feerate.h" -#include "protocol.h" -#include "random.h" -#include "streams.h" -#include "sync.h" -#include "uint256.h" -#include "threadinterrupt.h" +#include <addrdb.h> +#include <addrman.h> +#include <amount.h> +#include <bloom.h> +#include <compat.h> +#include <hash.h> +#include <limitedmap.h> +#include <netaddress.h> +#include <policy/feerate.h> +#include <protocol.h> +#include <random.h> +#include <streams.h> +#include <sync.h> +#include <uint256.h> +#include <threadinterrupt.h> #include <atomic> #include <deque> @@ -84,8 +84,6 @@ static const bool DEFAULT_FORCEDNSSEED = false; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; static const size_t DEFAULT_MAXSENDBUFFER = 1 * 1000; -static const ServiceFlags REQUIRED_SERVICES = NODE_NETWORK; - // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; // Default 24-hour ban @@ -130,7 +128,6 @@ public: struct Options { ServiceFlags nLocalServices = NODE_NONE; - ServiceFlags nRelevantServices = NODE_NONE; int nMaxConnections = 0; int nMaxOutbound = 0; int nMaxAddnode = 0; @@ -152,7 +149,6 @@ public: void Init(const Options& connOptions) { nLocalServices = connOptions.nLocalServices; - nRelevantServices = connOptions.nRelevantServices; nMaxConnections = connOptions.nMaxConnections; nMaxOutbound = std::min(connOptions.nMaxOutbound, connOptions.nMaxConnections); nMaxAddnode = connOptions.nMaxAddnode; @@ -162,10 +158,16 @@ public: m_msgproc = connOptions.m_msgproc; nSendBufferMaxSize = connOptions.nSendBufferMaxSize; nReceiveFloodSize = connOptions.nReceiveFloodSize; - nMaxOutboundTimeframe = connOptions.nMaxOutboundTimeframe; - nMaxOutboundLimit = connOptions.nMaxOutboundLimit; + { + LOCK(cs_totalBytesSent); + nMaxOutboundTimeframe = connOptions.nMaxOutboundTimeframe; + nMaxOutboundLimit = connOptions.nMaxOutboundLimit; + } vWhitelistedRange = connOptions.vWhitelistedRange; - vAddedNodes = connOptions.m_added_nodes; + { + LOCK(cs_vAddedNodes); + vAddedNodes = connOptions.m_added_nodes; + } } CConnman(uint64_t seed0, uint64_t seed1); @@ -175,7 +177,7 @@ public: void Interrupt(); bool GetNetworkActive() const { return fNetworkActive; }; void SetNetworkActive(bool active); - bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool fAddnode = false); + bool OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant *grantOutbound = nullptr, const char *strDest = nullptr, bool fOneShot = false, bool fFeeler = false, bool manual_connection = false); bool CheckIncomingNonce(uint64_t nonce); bool ForNode(NodeId id, std::function<bool(CNode* pnode)> func); @@ -255,6 +257,19 @@ public: void GetBanned(banmap_t &banmap); void SetBanned(const banmap_t &banmap); + // This allows temporarily exceeding nMaxOutbound, with the goal of finding + // a peer that is better than all our current peers. + void SetTryNewOutboundPeer(bool flag); + bool GetTryNewOutboundPeer(); + + // Return the number of outbound peers we have in excess of our target (eg, + // if we previously called SetTryNewOutboundPeer(true), and have since set + // to false, we may have extra peers that we wish to disconnect). This may + // return a value less than (num_outbound_connections - num_outbound_slots) + // in cases where some outbound connections are not yet fully connected, or + // not yet fully disconnected. + int GetExtraOutboundCount(); + bool AddNode(const std::string& node); bool RemoveAddedNode(const std::string& node); std::vector<AddedNodeInfo> GetAddedNodeInfo(); @@ -355,14 +370,14 @@ private: // Network usage totals CCriticalSection cs_totalBytesRecv; CCriticalSection cs_totalBytesSent; - uint64_t nTotalBytesRecv; - uint64_t nTotalBytesSent; + uint64_t nTotalBytesRecv GUARDED_BY(cs_totalBytesRecv); + uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent); // outbound limit & stats - uint64_t nMaxOutboundTotalBytesSentInCycle; - uint64_t nMaxOutboundCycleStartTime; - uint64_t nMaxOutboundLimit; - uint64_t nMaxOutboundTimeframe; + uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(cs_totalBytesSent); + uint64_t nMaxOutboundCycleStartTime GUARDED_BY(cs_totalBytesSent); + uint64_t nMaxOutboundLimit GUARDED_BY(cs_totalBytesSent); + uint64_t nMaxOutboundTimeframe GUARDED_BY(cs_totalBytesSent); // Whitelisted ranges. Any node connecting from these is automatically // whitelisted (as well as those connecting to whitelisted binds). @@ -380,7 +395,7 @@ private: CAddrMan addrman; std::deque<std::string> vOneShots; CCriticalSection cs_vOneShots; - std::vector<std::string> vAddedNodes; + std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); CCriticalSection cs_vAddedNodes; std::vector<CNode*> vNodes; std::list<CNode*> vNodesDisconnected; @@ -390,11 +405,8 @@ private: /** Services this instance offers */ ServiceFlags nLocalServices; - /** Services this instance cares about */ - ServiceFlags nRelevantServices; - - CSemaphore *semOutbound; - CSemaphore *semAddnode; + std::unique_ptr<CSemaphore> semOutbound; + std::unique_ptr<CSemaphore> semAddnode; int nMaxConnections; int nMaxOutbound; int nMaxAddnode; @@ -420,6 +432,13 @@ private: std::thread threadOpenAddedConnections; std::thread threadOpenConnections; std::thread threadMessageHandler; + + /** flag for deciding to connect to an extra outbound peer, + * in excess of nMaxOutbound + * This takes the place of a feeler connection */ + std::atomic_bool m_try_another_outbound_peer; + + friend struct CConnmanTest; }; extern std::unique_ptr<CConnman> g_connman; void Discover(boost::thread_group& threadGroup); @@ -513,7 +532,7 @@ public: int nVersion; std::string cleanSubVer; bool fInbound; - bool fAddnode; + bool m_manual_connection; int nStartingHeight; uint64_t nSendBytes; mapMsgCmdSize mapSendBytesPerMsgCmd; @@ -585,7 +604,6 @@ class CNode public: // socket std::atomic<ServiceFlags> nServices; - ServiceFlags nServicesExpected; SOCKET hSocket; size_t nSendSize; // total size of all vSendMsg entries size_t nSendOffset; // offset inside the first vSendMsg already sent @@ -623,7 +641,7 @@ public: bool fWhitelisted; // This peer can bypass DoS banning. bool fFeeler; // If true this node is being used as a short lived feeler. bool fOneShot; - bool fAddnode; + bool m_manual_connection; bool fClient; const bool fInbound; std::atomic_bool fSuccessfullyConnected; @@ -636,7 +654,7 @@ public: bool fSentAddr; CSemaphoreGrant grantOutbound; CCriticalSection cs_filter; - CBloomFilter* pfilter; + std::unique_ptr<CBloomFilter> pfilter; std::atomic<int> nRefCount; const uint64_t nKeyedNetGroup; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 7fced41d4f..3cf96be61a 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1,35 +1,34 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "net_processing.h" - -#include "addrman.h" -#include "arith_uint256.h" -#include "blockencodings.h" -#include "chainparams.h" -#include "consensus/validation.h" -#include "hash.h" -#include "init.h" -#include "validation.h" -#include "merkleblock.h" -#include "net.h" -#include "netmessagemaker.h" -#include "netbase.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "random.h" -#include "reverse_iterator.h" -#include "tinyformat.h" -#include "txmempool.h" -#include "ui_interface.h" -#include "util.h" -#include "utilmoneystr.h" -#include "utilstrencodings.h" -#include "validationinterface.h" +#include <net_processing.h> + +#include <addrman.h> +#include <arith_uint256.h> +#include <blockencodings.h> +#include <chainparams.h> +#include <consensus/validation.h> +#include <hash.h> +#include <init.h> +#include <validation.h> +#include <merkleblock.h> +#include <netmessagemaker.h> +#include <netbase.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <random.h> +#include <reverse_iterator.h> +#include <scheduler.h> +#include <tinyformat.h> +#include <txmempool.h> +#include <ui_interface.h> +#include <util.h> +#include <utilmoneystr.h> +#include <utilstrencodings.h> #if defined(NDEBUG) # error "Bitcoin cannot be compiled without assertions." @@ -40,7 +39,7 @@ std::atomic<int64_t> nTimeBestReceived(0); // Used only to inform the wallet of struct IteratorComparator { template<typename I> - bool operator()(const I& a, const I& b) + bool operator()(const I& a, const I& b) const { return &(*a) < &(*b); } @@ -52,15 +51,24 @@ struct COrphanTx { NodeId fromPeer; int64_t nTimeExpire; }; -std::map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(cs_main); -std::map<COutPoint, std::set<std::map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main); -void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +static CCriticalSection g_cs_orphans; +std::map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(g_cs_orphans); +std::map<COutPoint, std::set<std::map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(g_cs_orphans); +void EraseOrphansFor(NodeId peer); -static size_t vExtraTxnForCompactIt = 0; -static std::vector<std::pair<uint256, CTransactionRef>> vExtraTxnForCompact GUARDED_BY(cs_main); +static size_t vExtraTxnForCompactIt GUARDED_BY(g_cs_orphans) = 0; +static std::vector<std::pair<uint256, CTransactionRef>> vExtraTxnForCompact GUARDED_BY(g_cs_orphans); static const uint64_t RANDOMIZER_ID_ADDRESS_RELAY = 0x3cac0035b5866b90ULL; // SHA256("main address relay")[0:8] +/// Age after which a stale block will no longer be served if requested as +/// protection against fingerprinting. Set to one month, denominated in seconds. +static const int STALE_RELAY_AGE_LIMIT = 30 * 24 * 60 * 60; + +/// Age after which a block is considered historical for purposes of rate +/// limiting block relay. Set to one week, denominated in seconds. +static const int HISTORICAL_BLOCK_AGE = 7 * 24 * 60 * 60; + // Internal stuff namespace { /** Number of nodes with fSyncStarted. */ @@ -116,6 +124,12 @@ namespace { /** Number of peers from which we're downloading blocks. */ int nPeersWithValidatedDownloads = 0; + /** Number of outbound peers with m_chain_sync.m_protect. */ + int g_outbound_peers_with_protect_from_disconnect = 0; + + /** When our tip was last updated. */ + std::atomic<int64_t> g_last_tip_update(0); + /** Relay map, protected by cs_main. */ typedef std::map<uint256, CTransactionRef> MapRelay; MapRelay mapRelay; @@ -193,6 +207,36 @@ struct CNodeState { */ bool fSupportsDesiredCmpctVersion; + /** State used to enforce CHAIN_SYNC_TIMEOUT + * Only in effect for outbound, non-manual connections, with + * m_protect == false + * Algorithm: if a peer's best known block has less work than our tip, + * set a timeout CHAIN_SYNC_TIMEOUT seconds in the future: + * - If at timeout their best known block now has more work than our tip + * when the timeout was set, then either reset the timeout or clear it + * (after comparing against our current tip's work) + * - If at timeout their best known block still has less work than our + * tip did when the timeout was set, then send a getheaders message, + * and set a shorter timeout, HEADERS_RESPONSE_TIME seconds in future. + * If their best known block is still behind when that new timeout is + * reached, disconnect. + */ + struct ChainSyncTimeoutState { + //! A timeout used for checking whether our peer has sufficiently synced + int64_t m_timeout; + //! A header with the work we require on our peer's chain + const CBlockIndex * m_work_header; + //! After timeout is reached, set to true after sending getheaders + bool m_sent_getheaders; + //! Whether this peer is protected from disconnection due to a bad/slow chain + bool m_protect; + }; + + ChainSyncTimeoutState m_chain_sync; + + //! Time of last new block announcement + int64_t m_last_block_announcement; + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; nMisbehavior = 0; @@ -215,6 +259,8 @@ struct CNodeState { fHaveWitness = false; fWantsCmpctWitness = false; fSupportsDesiredCmpctVersion = false; + m_chain_sync = { 0, nullptr, false, false }; + m_last_block_announcement = 0; } }; @@ -371,25 +417,32 @@ void MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid, CConnman* connman) { } } connman->ForNode(nodeid, [connman](CNode* pfrom){ - bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = (pfrom->GetLocalServices() & NODE_WITNESS) ? 2 : 1; if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. - connman->ForNode(lNodesAnnouncingHeaderAndIDs.front(), [connman, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion](CNode* pnodeStop){ - connman->PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->ForNode(lNodesAnnouncingHeaderAndIDs.front(), [connman, nCMPCTBLOCKVersion](CNode* pnodeStop){ + connman->PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/false, nCMPCTBLOCKVersion)); return true; }); lNodesAnnouncingHeaderAndIDs.pop_front(); } - fAnnounceUsingCMPCTBLOCK = true; - connman->PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + connman->PushMessage(pfrom, CNetMsgMaker(pfrom->GetSendVersion()).Make(NetMsgType::SENDCMPCT, /*fAnnounceUsingCMPCTBLOCK=*/true, nCMPCTBLOCKVersion)); lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); return true; }); } } +bool TipMayBeStale(const Consensus::Params &consensusParams) +{ + AssertLockHeld(cs_main); + if (g_last_tip_update == 0) { + g_last_tip_update = GetTime(); + } + return g_last_tip_update < GetTime() - consensusParams.nPowTargetSpacing * 3 && mapBlocksInFlight.empty(); +} + // Requires cs_main bool CanDirectFetch(const Consensus::Params &consensusParams) { @@ -496,6 +549,22 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<con } // namespace +// This function is used for testing the stale tip eviction logic, see +// DoS_tests.cpp +void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) +{ + LOCK(cs_main); + CNodeState *state = State(node); + if (state) state->m_last_block_announcement = time_in_seconds; +} + +// Returns true for outbound peers, excluding manual connections, feelers, and +// one-shots +bool IsOutboundDisconnectionCandidate(const CNode *node) +{ + return !(node->fInbound || node->m_manual_connection || node->fFeeler || node->fOneShot); +} + void PeerLogicValidation::InitializeNode(CNode *pnode) { CAddress addr = pnode->addr; std::string addrName = pnode->GetAddrName(); @@ -528,6 +597,8 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim nPreferredDownload -= state->fPreferredDownload; nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); assert(nPeersWithValidatedDownloads >= 0); + g_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; + assert(g_outbound_peers_with_protect_from_disconnect >= 0); mapNodeState.erase(nodeid); @@ -536,6 +607,7 @@ void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTim assert(mapBlocksInFlight.empty()); assert(nPreferredDownload == 0); assert(nPeersWithValidatedDownloads == 0); + assert(g_outbound_peers_with_protect_from_disconnect == 0); } LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } @@ -560,7 +632,7 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) { // mapOrphanTransactions // -void AddToCompactExtraTransactions(const CTransactionRef& tx) +void AddToCompactExtraTransactions(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) { size_t max_extra_txn = gArgs.GetArg("-blockreconstructionextratxn", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN); if (max_extra_txn <= 0) @@ -571,7 +643,7 @@ void AddToCompactExtraTransactions(const CTransactionRef& tx) vExtraTxnForCompactIt = (vExtraTxnForCompactIt + 1) % max_extra_txn; } -bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) { const uint256& hash = tx->GetHash(); if (mapOrphanTransactions.count(hash)) @@ -604,7 +676,7 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE return true; } -int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) { std::map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.find(hash); if (it == mapOrphanTransactions.end()) @@ -624,6 +696,7 @@ int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) void EraseOrphansFor(NodeId peer) { + LOCK(g_cs_orphans); int nErased = 0; std::map<uint256, COrphanTx>::iterator iter = mapOrphanTransactions.begin(); while (iter != mapOrphanTransactions.end()) @@ -638,8 +711,10 @@ void EraseOrphansFor(NodeId peer) } -unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) { + LOCK(g_cs_orphans); + unsigned int nEvicted = 0; static int64_t nNextSweep; int64_t nNow = GetTime(); @@ -706,13 +781,34 @@ void Misbehaving(NodeId pnode, int howmuch) // blockchain -> download logic notification // -PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn) : connman(connmanIn) { +// To prevent fingerprinting attacks, only send blocks/headers outside of the +// active chain if they are no more than a month older (both in time, and in +// best equivalent proof of work) than the best header chain we know about and +// we fully-validated them at some point. +static bool BlockRequestAllowed(const CBlockIndex* pindex, const Consensus::Params& consensusParams) +{ + AssertLockHeld(cs_main); + if (chainActive.Contains(pindex)) return true; + return pindex->IsValid(BLOCK_VALID_SCRIPTS) && (pindexBestHeader != nullptr) && + (pindexBestHeader->GetBlockTime() - pindex->GetBlockTime() < STALE_RELAY_AGE_LIMIT) && + (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, consensusParams) < STALE_RELAY_AGE_LIMIT); +} + +PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, CScheduler &scheduler) : connman(connmanIn), m_stale_tip_check_time(0) { // Initialize global variables that cannot be constructed at startup. recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); + + const Consensus::Params& consensusParams = Params().GetConsensus(); + // Stale tip checking and peer eviction are on two different timers, but we + // don't want them to get out of sync due to drift in the scheduler, so we + // combine them in one function and schedule at the quicker (peer-eviction) + // timer. + static_assert(EXTRA_PEER_CHECK_INTERVAL < STALE_CHECK_INTERVAL, "peer eviction timer should be less than stale tip check timer"); + scheduler.scheduleEvery(std::bind(&PeerLogicValidation::CheckForStaleTipAndEvictPeers, this, consensusParams), EXTRA_PEER_CHECK_INTERVAL * 1000); } void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex, const std::vector<CTransactionRef>& vtxConflicted) { - LOCK(cs_main); + LOCK(g_cs_orphans); std::vector<uint256> vOrphanErase; @@ -739,6 +835,8 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pb } LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } + + g_last_tip_update = GetTime(); } // All of the following cache a recent block, and are protected by cs_most_recent_block @@ -877,9 +975,13 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) recentRejects->reset(); } + { + LOCK(g_cs_orphans); + if (mapOrphanTransactions.count(inv.hash)) return true; + } + return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || - mapOrphanTransactions.count(inv.hash) || pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)); } @@ -936,181 +1038,198 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connma connman->ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); } +void static ProcessGetBlockData(CNode* pfrom, const Consensus::Params& consensusParams, const CInv& inv, CConnman* connman, const std::atomic<bool>& interruptMsgProc) +{ + bool send = false; + std::shared_ptr<const CBlock> a_recent_block; + std::shared_ptr<const CBlockHeaderAndShortTxIDs> a_recent_compact_block; + bool fWitnessesPresentInARecentCompactBlock; + { + LOCK(cs_most_recent_block); + a_recent_block = most_recent_block; + a_recent_compact_block = most_recent_compact_block; + fWitnessesPresentInARecentCompactBlock = fWitnessesPresentInMostRecentCompactBlock; + } + + bool need_activate_chain = false; + { + LOCK(cs_main); + BlockMap::iterator mi = mapBlockIndex.find(inv.hash); + if (mi != mapBlockIndex.end()) + { + if (mi->second->nChainTx && !mi->second->IsValid(BLOCK_VALID_SCRIPTS) && + mi->second->IsValid(BLOCK_VALID_TREE)) { + // If we have the block and all of its parents, but have not yet validated it, + // we might be in the middle of connecting it (ie in the unlock of cs_main + // before ActivateBestChain but after AcceptBlock). + // In this case, we need to run ActivateBestChain prior to checking the relay + // conditions below. + need_activate_chain = true; + } + } + } // release cs_main before calling ActivateBestChain + if (need_activate_chain) { + CValidationState dummy; + ActivateBestChain(dummy, Params(), a_recent_block); + } + + LOCK(cs_main); + BlockMap::iterator mi = mapBlockIndex.find(inv.hash); + if (mi != mapBlockIndex.end()) { + send = BlockRequestAllowed(mi->second, consensusParams); + if (!send) { + LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom->GetId()); + } + } + const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + // disconnect node in case we have reached the outbound limit for serving historical blocks + // never disconnect whitelisted nodes + if (send && connman->OutboundTargetReached(true) && ( ((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) + { + LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); + + //disconnect node + pfrom->fDisconnect = true; + send = false; + } + // Avoid leaking prune-height by never sending blocks below the NODE_NETWORK_LIMITED threshold + if (send && !pfrom->fWhitelisted && ( + (((pfrom->GetLocalServices() & NODE_NETWORK_LIMITED) == NODE_NETWORK_LIMITED) && ((pfrom->GetLocalServices() & NODE_NETWORK) != NODE_NETWORK) && (chainActive.Tip()->nHeight - mi->second->nHeight > (int)NODE_NETWORK_LIMITED_MIN_BLOCKS + 2 /* add two blocks buffer extension for possible races */) ) + )) { + LogPrint(BCLog::NET, "Ignore block request below NODE_NETWORK_LIMITED threshold from peer=%d\n", pfrom->GetId()); + + //disconnect node and prevent it from stalling (would otherwise wait for the missing block) + pfrom->fDisconnect = true; + send = false; + } + // Pruned nodes may have deleted the block, so check whether + // it's available before trying to send. + if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) + { + std::shared_ptr<const CBlock> pblock; + if (a_recent_block && a_recent_block->GetHash() == (*mi).second->GetBlockHash()) { + pblock = a_recent_block; + } else { + // Send block from disk + std::shared_ptr<CBlock> pblockRead = std::make_shared<CBlock>(); + if (!ReadBlockFromDisk(*pblockRead, (*mi).second, consensusParams)) + assert(!"cannot load block from disk"); + pblock = pblockRead; + } + if (inv.type == MSG_BLOCK) + connman->PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); + else if (inv.type == MSG_WITNESS_BLOCK) + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + else if (inv.type == MSG_FILTERED_BLOCK) + { + bool sendMerkleBlock = false; + CMerkleBlock merkleBlock; + { + LOCK(pfrom->cs_filter); + if (pfrom->pfilter) { + sendMerkleBlock = true; + merkleBlock = CMerkleBlock(*pblock, *pfrom->pfilter); + } + } + if (sendMerkleBlock) { + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); + // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see + // This avoids hurting performance by pointlessly requiring a round-trip + // Note that there is currently no way for a node to request any single transactions we didn't send here - + // they must either disconnect and retry or request the full block. + // Thus, the protocol spec specified allows for us to provide duplicate txn here, + // however we MUST always provide at least what the remote peer needs + typedef std::pair<unsigned int, uint256> PairType; + for (PairType& pair : merkleBlock.vMatchedTxn) + connman->PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); + } + // else + // no response + } + else if (inv.type == MSG_CMPCT_BLOCK) + { + // If a peer is asking for old blocks, we're almost guaranteed + // they won't have a useful mempool to match against a compact block, + // and we don't feel like constructing the object for them, so + // instead we respond with the full, non-compact block. + bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; + int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; + if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { + if ((fPeerWantsWitness || !fWitnessesPresentInARecentCompactBlock) && a_recent_compact_block && a_recent_compact_block->header.GetHash() == mi->second->GetBlockHash()) { + connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + } else { + CBlockHeaderAndShortTxIDs cmpctblock(*pblock, fPeerWantsWitness); + connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + } + } else { + connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); + } + } + + // Trigger the peer node to send a getblocks request for the next batch of inventory + if (inv.hash == pfrom->hashContinue) + { + // Bypass PushInventory, this must send even if redundant, + // and we want it right after the last block so they don't + // wait for other stuff first. + std::vector<CInv> vInv; + vInv.push_back(CInv(MSG_BLOCK, chainActive.Tip()->GetBlockHash())); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + pfrom->hashContinue.SetNull(); + } + } +} + void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParams, CConnman* connman, const std::atomic<bool>& interruptMsgProc) { + AssertLockNotHeld(cs_main); + std::deque<CInv>::iterator it = pfrom->vRecvGetData.begin(); std::vector<CInv> vNotFound; const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - LOCK(cs_main); - - while (it != pfrom->vRecvGetData.end()) { - // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - break; + { + LOCK(cs_main); - const CInv &inv = *it; - { + while (it != pfrom->vRecvGetData.end() && (it->type == MSG_TX || it->type == MSG_WITNESS_TX)) { if (interruptMsgProc) return; + // Don't bother if send buffer is too full to respond anyway + if (pfrom->fPauseSend) + break; + const CInv &inv = *it; it++; - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) - { - bool send = false; - BlockMap::iterator mi = mapBlockIndex.find(inv.hash); - std::shared_ptr<const CBlock> a_recent_block; - std::shared_ptr<const CBlockHeaderAndShortTxIDs> a_recent_compact_block; - bool fWitnessesPresentInARecentCompactBlock; - { - LOCK(cs_most_recent_block); - a_recent_block = most_recent_block; - a_recent_compact_block = most_recent_compact_block; - fWitnessesPresentInARecentCompactBlock = fWitnessesPresentInMostRecentCompactBlock; - } - if (mi != mapBlockIndex.end()) - { - if (mi->second->nChainTx && !mi->second->IsValid(BLOCK_VALID_SCRIPTS) && - mi->second->IsValid(BLOCK_VALID_TREE)) { - // If we have the block and all of its parents, but have not yet validated it, - // we might be in the middle of connecting it (ie in the unlock of cs_main - // before ActivateBestChain but after AcceptBlock). - // In this case, we need to run ActivateBestChain prior to checking the relay - // conditions below. - CValidationState dummy; - ActivateBestChain(dummy, Params(), a_recent_block); - } - if (chainActive.Contains(mi->second)) { - send = true; - } else { - static const int nOneMonth = 30 * 24 * 60 * 60; - // To prevent fingerprinting attacks, only send blocks outside of the active - // chain if they are valid, and no more than a month older (both in time, and in - // best equivalent proof of work) than the best header chain we know about. - send = mi->second->IsValid(BLOCK_VALID_SCRIPTS) && (pindexBestHeader != nullptr) && - (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() < nOneMonth) && - (GetBlockProofEquivalentTime(*pindexBestHeader, *mi->second, *pindexBestHeader, consensusParams) < nOneMonth); - if (!send) { - LogPrintf("%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom->GetId()); - } - } - } - // disconnect node in case we have reached the outbound limit for serving historical blocks - // never disconnect whitelisted nodes - static const int nOneWeek = 7 * 24 * 60 * 60; // assume > 1 week = historical - if (send && connman->OutboundTargetReached(true) && ( ((pindexBestHeader != nullptr) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > nOneWeek)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) - { - LogPrint(BCLog::NET, "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); - - //disconnect node - pfrom->fDisconnect = true; - send = false; - } - // Pruned nodes may have deleted the block, so check whether - // it's available before trying to send. - if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) - { - std::shared_ptr<const CBlock> pblock; - if (a_recent_block && a_recent_block->GetHash() == (*mi).second->GetBlockHash()) { - pblock = a_recent_block; - } else { - // Send block from disk - std::shared_ptr<CBlock> pblockRead = std::make_shared<CBlock>(); - if (!ReadBlockFromDisk(*pblockRead, (*mi).second, consensusParams)) - assert(!"cannot load block from disk"); - pblock = pblockRead; - } - if (inv.type == MSG_BLOCK) - connman->PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); - else if (inv.type == MSG_WITNESS_BLOCK) - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); - else if (inv.type == MSG_FILTERED_BLOCK) - { - bool sendMerkleBlock = false; - CMerkleBlock merkleBlock; - { - LOCK(pfrom->cs_filter); - if (pfrom->pfilter) { - sendMerkleBlock = true; - merkleBlock = CMerkleBlock(*pblock, *pfrom->pfilter); - } - } - if (sendMerkleBlock) { - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); - // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see - // This avoids hurting performance by pointlessly requiring a round-trip - // Note that there is currently no way for a node to request any single transactions we didn't send here - - // they must either disconnect and retry or request the full block. - // Thus, the protocol spec specified allows for us to provide duplicate txn here, - // however we MUST always provide at least what the remote peer needs - typedef std::pair<unsigned int, uint256> PairType; - for (PairType& pair : merkleBlock.vMatchedTxn) - connman->PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); - } - // else - // no response - } - else if (inv.type == MSG_CMPCT_BLOCK) - { - // If a peer is asking for old blocks, we're almost guaranteed - // they won't have a useful mempool to match against a compact block, - // and we don't feel like constructing the object for them, so - // instead we respond with the full, non-compact block. - bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; - int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; - if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { - if ((fPeerWantsWitness || !fWitnessesPresentInARecentCompactBlock) && a_recent_compact_block && a_recent_compact_block->header.GetHash() == mi->second->GetBlockHash()) { - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); - } else { - CBlockHeaderAndShortTxIDs cmpctblock(*pblock, fPeerWantsWitness); - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); - } - } else { - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); - } - } - - // Trigger the peer node to send a getblocks request for the next batch of inventory - if (inv.hash == pfrom->hashContinue) - { - // Bypass PushInventory, this must send even if redundant, - // and we want it right after the last block so they don't - // wait for other stuff first. - std::vector<CInv> vInv; - vInv.push_back(CInv(MSG_BLOCK, chainActive.Tip()->GetBlockHash())); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::INV, vInv)); - pfrom->hashContinue.SetNull(); - } - } - } - else if (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) - { - // Send stream from relay memory - bool push = false; - auto mi = mapRelay.find(inv.hash); - int nSendFlags = (inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); - if (mi != mapRelay.end()) { - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *mi->second)); + // Send stream from relay memory + bool push = false; + auto mi = mapRelay.find(inv.hash); + int nSendFlags = (inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); + if (mi != mapRelay.end()) { + connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *mi->second)); + push = true; + } else if (pfrom->timeLastMempoolReq) { + auto txinfo = mempool.info(inv.hash); + // To protect privacy, do not answer getdata using the mempool when + // that TX couldn't have been INVed in reply to a MEMPOOL request. + if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { + connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *txinfo.tx)); push = true; - } else if (pfrom->timeLastMempoolReq) { - auto txinfo = mempool.info(inv.hash); - // To protect privacy, do not answer getdata using the mempool when - // that TX couldn't have been INVed in reply to a MEMPOOL request. - if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { - connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *txinfo.tx)); - push = true; - } - } - if (!push) { - vNotFound.push_back(inv); } } + if (!push) { + vNotFound.push_back(inv); + } // Track requests for our stuff. GetMainSignals().Inventory(inv.hash); + } + } // release cs_main - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) - break; + if (it != pfrom->vRecvGetData.end()) { + const CInv &inv = *it; + it++; + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) { + ProcessGetBlockData(pfrom, consensusParams, inv, connman, interruptMsgProc); } } @@ -1153,6 +1272,225 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac connman->PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCKTXN, resp)); } +bool static ProcessHeadersMessage(CNode *pfrom, CConnman *connman, const std::vector<CBlockHeader>& headers, const CChainParams& chainparams, bool punish_duplicate_invalid) +{ + const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); + size_t nCount = headers.size(); + + if (nCount == 0) { + // Nothing interesting. Stop asking this peers for more headers. + return true; + } + + bool received_new_header = false; + const CBlockIndex *pindexLast = nullptr; + { + LOCK(cs_main); + CNodeState *nodestate = State(pfrom->GetId()); + + // If this looks like it could be a block announcement (nCount < + // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that + // don't connect: + // - Send a getheaders message in response to try to connect the chain. + // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + // don't connect before giving DoS points + // - Once a headers message is received that is valid and does connect, + // nUnconnectingHeaders gets reset back to 0. + if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { + nodestate->nUnconnectingHeaders++; + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + pindexBestHeader->nHeight, + pfrom->GetId(), nodestate->nUnconnectingHeaders); + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); + + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(pfrom->GetId(), 20); + } + return true; + } + + uint256 hashLastBlock; + for (const CBlockHeader& header : headers) { + if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { + Misbehaving(pfrom->GetId(), 20); + return error("non-continuous headers sequence"); + } + hashLastBlock = header.GetHash(); + } + + // If we don't have the last header, then they'll have given us + // something new (if these headers are valid). + if (mapBlockIndex.find(hashLastBlock) == mapBlockIndex.end()) { + received_new_header = true; + } + } + + CValidationState state; + CBlockHeader first_invalid_header; + if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast, &first_invalid_header)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + LOCK(cs_main); + if (nDoS > 0) { + Misbehaving(pfrom->GetId(), nDoS); + } + if (punish_duplicate_invalid && mapBlockIndex.find(first_invalid_header.GetHash()) != mapBlockIndex.end()) { + // Goal: don't allow outbound peers to use up our outbound + // connection slots if they are on incompatible chains. + // + // We ask the caller to set punish_invalid appropriately based + // on the peer and the method of header delivery (compact + // blocks are allowed to be invalid in some circumstances, + // under BIP 152). + // Here, we try to detect the narrow situation that we have a + // valid block header (ie it was valid at the time the header + // was received, and hence stored in mapBlockIndex) but know the + // block is invalid, and that a peer has announced that same + // block as being on its active chain. + // Disconnect the peer in such a situation. + // + // Note: if the header that is invalid was not accepted to our + // mapBlockIndex at all, that may also be grounds for + // disconnecting the peer, as the chain they are on is likely + // to be incompatible. However, there is a circumstance where + // that does not hold: if the header's timestamp is more than + // 2 hours ahead of our current time. In that case, the header + // may become valid in the future, and we don't want to + // disconnect a peer merely for serving us one too-far-ahead + // block header, to prevent an attacker from splitting the + // network by mining a block right at the 2 hour boundary. + // + // TODO: update the DoS logic (or, rather, rewrite the + // DoS-interface between validation and net_processing) so that + // the interface is cleaner, and so that we disconnect on all the + // reasons that a peer's headers chain is incompatible + // with ours (eg block->nVersion softforks, MTP violations, + // etc), and not just the duplicate-invalid case. + pfrom->fDisconnect = true; + } + return error("invalid header received"); + } + } + + { + LOCK(cs_main); + CNodeState *nodestate = State(pfrom->GetId()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + + // From here, pindexBestKnownBlock should be guaranteed to be non-null, + // because it is set in UpdateBlockAvailability. Some nullptr checks + // are still present, however, as belt-and-suspenders. + + if (received_new_header && pindexLast->nChainWork > chainActive.Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + + if (nCount == MAX_HEADERS_RESULTS) { + // Headers message had its maximum size; the peer may have more headers. + // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue + // from there instead. + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); + } + + bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); + // If this set of headers is valid and ends in a block with at least as + // much work as our tip, download as much as possible. + if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { + std::vector<const CBlockIndex*> vToFetch; + const CBlockIndex *pindexWalk = pindexLast; + // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && + !mapBlocksInFlight.count(pindexWalk->GetBlockHash()) && + (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { + // We don't have this block, and it's not yet in flight. + vToFetch.push_back(pindexWalk); + } + pindexWalk = pindexWalk->pprev; + } + // If pindexWalk still isn't on our main chain, we're looking at a + // very large reorg at a time we think we're close to caught up to + // the main chain -- this shouldn't really happen. Bail out on the + // direct fetch and rely on parallel download instead. + if (!chainActive.Contains(pindexWalk)) { + LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", + pindexLast->GetBlockHash().ToString(), + pindexLast->nHeight); + } else { + std::vector<CInv> vGetData; + // Download as much as possible, from earliest to latest. + for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { + if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + // Can't download any more from this peer + break; + } + uint32_t nFetchFlags = GetFetchFlags(pfrom); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); + LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom->GetId()); + } + if (vGetData.size() > 1) { + LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", + pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + } + if (vGetData.size() > 0) { + if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } + connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + } + } + } + // If we're in IBD, we want outbound peers that will serve us a useful + // chain. Disconnect peers that are on chains with insufficient work. + if (IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { + // When nCount < MAX_HEADERS_RESULTS, we know we have no more + // headers to fetch from this peer. + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + // This peer has too little work on their headers chain to help + // us sync -- disconnect if using an outbound slot (unless + // whitelisted or addnode). + // Note: We compare their tip to nMinimumChainWork (rather than + // chainActive.Tip()) because we won't start block download + // until we have a headers chain that has at least + // nMinimumChainWork, even if a peer has a chain past our tip, + // as an anti-DoS measure. + if (IsOutboundDisconnectionCandidate(pfrom)) { + LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom->GetId()); + pfrom->fDisconnect = true; + } + } + } + + if (!pfrom->fDisconnect && IsOutboundDisconnectionCandidate(pfrom) && nodestate->pindexBestKnownBlock != nullptr) { + // If this is an outbound peer, check to see if we should protect + // it from the bad/lagging chain logic. + if (g_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom->GetId()); + nodestate->m_chain_sync.m_protect = true; + ++g_outbound_peers_with_protect_from_disconnect; + } + } + } + + return true; +} + bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman* connman, const std::atomic<bool>& interruptMsgProc) { LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); @@ -1232,11 +1570,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { connman->SetServices(pfrom->addr, nServices); } - if (pfrom->nServicesExpected & ~nServices) + if (!pfrom->fInbound && !pfrom->fFeeler && !pfrom->m_manual_connection && !HasAllDesirableServiceFlags(nServices)) { - LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, pfrom->nServicesExpected); + LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, GetDesirableServiceFlags(nServices)); connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, - strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); + strprintf("Expected to offer services %08x", GetDesirableServiceFlags(nServices)))); pfrom->fDisconnect = true; return false; } @@ -1255,7 +1593,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version - LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); + LogPrint(BCLog::NET, "peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); connman->PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); pfrom->fDisconnect = true; @@ -1355,7 +1693,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (fLogIPs) remoteAddr = ", peeraddr=" + pfrom->addr.ToString(); - LogPrintf("receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", + LogPrint(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", cleanSubVer, pfrom->nVersion, pfrom->nStartingHeight, addrMe.ToString(), pfrom->GetId(), remoteAddr); @@ -1398,6 +1736,9 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Mark this node as currently connected, so we update its timestamp later. LOCK(cs_main); State(pfrom->GetId())->fCurrentlyConnected = true; + LogPrintf("New outbound peer connected: version: %d, blocks=%d, peer=%d%s\n", + pfrom->nVersion.load(), pfrom->nStartingHeight, pfrom->GetId(), + (fLogIPs ? strprintf(", peeraddr=%s", pfrom->addr.ToString()) : "")); } if (pfrom->nVersion >= SENDHEADERS_VERSION) { @@ -1455,7 +1796,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (interruptMsgProc) return true; - if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) + // We only bother storing full nodes, though this may include + // things which we would not make an outbound connection to, in + // part because we may make feeler connections to them. + if (!MayHaveUsefulAddressDB(addr.nServices)) continue; if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) @@ -1673,7 +2017,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { - LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->GetId()); + LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block we don't have", pfrom->GetId()); return true; } @@ -1690,7 +2034,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr inv.type = State(pfrom->GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; inv.hash = req.blockhash; pfrom->vRecvGetData.push_back(inv); - ProcessGetData(pfrom, chainparams.GetConsensus(), connman, interruptMsgProc); + // The message processing loop will go around again (without pausing) and we'll respond then (without cs_main) return true; } @@ -1723,6 +2067,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (mi == mapBlockIndex.end()) return true; pindex = (*mi).second; + + if (!BlockRequestAllowed(pindex, chainparams.GetConsensus())) { + LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block header that isn't in the main chain\n", __func__, pfrom->GetId()); + return true; + } } else { @@ -1778,7 +2127,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr CInv inv(MSG_TX, tx.GetHash()); pfrom->AddInventoryKnown(inv); - LOCK(cs_main); + LOCK2(cs_main, g_cs_orphans); bool fMissingInputs = false; CValidationState state; @@ -1790,7 +2139,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, ptx, &fMissingInputs, &lRemovedTxn, false /* bypass_limits */, 0 /* nAbsurdFee */)) { - mempool.check(pcoinsTip); + mempool.check(pcoinsTip.get()); RelayTransaction(tx, connman); for (unsigned int i = 0; i < tx.vout.size(); i++) { vWorkQueue.emplace_back(inv.hash, i); @@ -1857,7 +2206,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr recentRejects->insert(orphanHash); } } - mempool.check(pcoinsTip); + mempool.check(pcoinsTip.get()); } } @@ -1951,6 +2300,8 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr CBlockHeaderAndShortTxIDs cmpctblock; vRecv >> cmpctblock; + bool received_new_header = false; + { LOCK(cs_main); @@ -1960,6 +2311,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); return true; } + + if (mapBlockIndex.find(cmpctblock.header.GetHash()) == mapBlockIndex.end()) { + received_new_header = true; + } } const CBlockIndex *pindex = nullptr; @@ -1968,10 +2323,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr int nDoS; if (state.IsInvalid(nDoS)) { if (nDoS > 0) { + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); + } else { + LogPrint(BCLog::NET, "Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); } - LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); return true; } } @@ -1986,7 +2343,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // If we end up treating this as a plain headers message, call that as well // without cs_main. bool fRevertToHeaderProcessing = false; - CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); // Keep a CBlock for "optimistic" compactblock reconstructions (see // below) @@ -1994,11 +2350,19 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr bool fBlockReconstructed = false; { - LOCK(cs_main); + LOCK2(cs_main, g_cs_orphans); // If AcceptBlockHeader returned true, it set pindex assert(pindex); UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); + CNodeState *nodestate = State(pfrom->GetId()); + + // If this was a new header with more work than our tip, update the + // peer's last block announcement time + if (received_new_header && pindex->nChainWork > chainActive.Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); @@ -2021,8 +2385,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) return true; - CNodeState *nodestate = State(pfrom->GetId()); - if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus()) && !nodestate->fSupportsDesiredCmpctVersion) { // Don't bother trying to process compact blocks from v1 peers // after segwit activates. @@ -2103,10 +2465,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return true; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message - // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) - std::vector<CBlock> headers; - headers.push_back(cmpctblock.header); - vHeadersMsg << headers; fRevertToHeaderProcessing = true; } } @@ -2115,8 +2473,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (fProcessBLOCKTXN) return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams, connman, interruptMsgProc); - if (fRevertToHeaderProcessing) - return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams, connman, interruptMsgProc); + if (fRevertToHeaderProcessing) { + // Headers received from HB compact block peers are permitted to be + // relayed before full validation (see BIP 152), so we don't want to disconnect + // the peer if the header turns out to be for an invalid block. + // Note that if a peer tries to build on an invalid chain, that + // will be detected and the peer will be banned. + return ProcessHeadersMessage(pfrom, connman, {cmpctblock.header}, chainparams, /*punish_duplicate_invalid=*/false); + } if (fBlockReconstructed) { // If we got here, we were able to optimistically reconstruct a @@ -2126,7 +2490,16 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr mapBlockSource.emplace(pblock->GetHash(), std::make_pair(pfrom->GetId(), false)); } bool fNewBlock = false; - ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + // Setting fForceProcessing to true means that we bypass some of + // our anti-DoS protections in AcceptBlock, which filters + // unrequested blocks that might be trying to waste our resources + // (eg disk space). Because we only try to reconstruct blocks when + // we're close to caught up (via the CanDirectFetch() requirement + // above, combined with the behavior of not requesting blocks until + // we have a chain with at least nMinimumChainWork), and we ignore + // compact blocks with less work than our tip, it is safe to treat + // reconstructed compact blocks as having been requested. + ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { @@ -2206,7 +2579,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr bool fNewBlock = false; // Since we requested this block (it was in mapBlocksInFlight), force it to be processed, // even if it would not be a candidate for new tip (missing previous block, chain not long enough, etc) - ProcessNewBlock(chainparams, pblock, true, &fNewBlock); + // This bypasses some anti-DoS logic in AcceptBlock (eg to prevent + // disk-space attacks), but this should be safe due to the + // protections in the compact block handler -- see related comment + // in compact block optimistic reconstruction handling. + ProcessNewBlock(chainparams, pblock, /*fForceProcessing=*/true, &fNewBlock); if (fNewBlock) { pfrom->nLastBlockTime = GetTime(); } else { @@ -2234,136 +2611,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } - if (nCount == 0) { - // Nothing interesting. Stop asking this peers for more headers. - return true; - } - - const CBlockIndex *pindexLast = nullptr; - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); - - // If this looks like it could be a block announcement (nCount < - // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that - // don't connect: - // - Send a getheaders message in response to try to connect the chain. - // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that - // don't connect before giving DoS points - // - Once a headers message is received that is valid and does connect, - // nUnconnectingHeaders gets reset back to 0. - if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { - nodestate->nUnconnectingHeaders++; - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256())); - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", - headers[0].GetHash().ToString(), - headers[0].hashPrevBlock.ToString(), - pindexBestHeader->nHeight, - pfrom->GetId(), nodestate->nUnconnectingHeaders); - // Set hashLastUnknownBlock for this peer, so that if we - // eventually get the headers - even from a different peer - - // we can use this peer to download. - UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); - - if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(pfrom->GetId(), 20); - } - return true; - } - - uint256 hashLastBlock; - for (const CBlockHeader& header : headers) { - if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { - Misbehaving(pfrom->GetId(), 20); - return error("non-continuous headers sequence"); - } - hashLastBlock = header.GetHash(); - } - } - - CValidationState state; - if (!ProcessNewBlockHeaders(headers, state, chainparams, &pindexLast)) { - int nDoS; - if (state.IsInvalid(nDoS)) { - if (nDoS > 0) { - LOCK(cs_main); - Misbehaving(pfrom->GetId(), nDoS); - } - return error("invalid header received"); - } - } - - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom->GetId()); - if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); - } - nodestate->nUnconnectingHeaders = 0; - - assert(pindexLast); - UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); - - if (nCount == MAX_HEADERS_RESULTS) { - // Headers message had its maximum size; the peer may have more headers. - // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue - // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); - } - - bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); - // If this set of headers is valid and ends in a block with at least as - // much work as our tip, download as much as possible. - if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { - std::vector<const CBlockIndex*> vToFetch; - const CBlockIndex *pindexWalk = pindexLast; - // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. - while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && - !mapBlocksInFlight.count(pindexWalk->GetBlockHash()) && - (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { - // We don't have this block, and it's not yet in flight. - vToFetch.push_back(pindexWalk); - } - pindexWalk = pindexWalk->pprev; - } - // If pindexWalk still isn't on our main chain, we're looking at a - // very large reorg at a time we think we're close to caught up to - // the main chain -- this shouldn't really happen. Bail out on the - // direct fetch and rely on parallel download instead. - if (!chainActive.Contains(pindexWalk)) { - LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", - pindexLast->GetBlockHash().ToString(), - pindexLast->nHeight); - } else { - std::vector<CInv> vGetData; - // Download as much as possible, from earliest to latest. - for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - // Can't download any more from this peer - break; - } - uint32_t nFetchFlags = GetFetchFlags(pfrom); - vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); - LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom->GetId()); - } - if (vGetData.size() > 1) { - LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", - pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); - } - if (vGetData.size() > 0) { - if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // In any case, we want to download using a compact block, not a regular one - vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); - } - connman->PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - } - } - } - } + // Headers received via a HEADERS message should be valid, and reflect + // the chain the peer is on. If we receive a known-invalid header, + // disconnect the peer if it is using one of our outbound connection + // slots. + bool should_punish = !pfrom->fInbound && !pfrom->m_manual_connection; + return ProcessHeadersMessage(pfrom, connman, headers, chainparams, should_punish); } else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing @@ -2373,11 +2626,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->GetId()); - // Process all blocks from whitelisted peers, even if not requested, - // unless we're still syncing with the network. - // Such an unrequested block may still be processed, subject to the - // conditions in AcceptBlock(). - bool forceProcessing = pfrom->fWhitelisted && !IsInitialBlockDownload(); + bool forceProcessing = false; const uint256 hash(pblock->GetHash()); { LOCK(cs_main); @@ -2541,8 +2790,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr else { LOCK(pfrom->cs_filter); - delete pfrom->pfilter; - pfrom->pfilter = new CBloomFilter(filter); + pfrom->pfilter.reset(new CBloomFilter(filter)); pfrom->pfilter->UpdateEmptyFull(); pfrom->fRelayTxes = true; } @@ -2578,8 +2826,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr { LOCK(pfrom->cs_filter); if (pfrom->GetLocalServices() & NODE_BLOOM) { - delete pfrom->pfilter; - pfrom->pfilter = new CBloomFilter(); + pfrom->pfilter.reset(new CBloomFilter()); } pfrom->fRelayTxes = true; } @@ -2625,8 +2872,8 @@ static bool SendRejectsAndCheckIfBanned(CNode* pnode, CConnman* connman) state.fShouldBan = false; if (pnode->fWhitelisted) LogPrintf("Warning: not punishing whitelisted peer %s!\n", pnode->addr.ToString()); - else if (pnode->fAddnode) - LogPrintf("Warning: not punishing addnoded peer %s!\n", pnode->addr.ToString()); + else if (pnode->m_manual_connection) + LogPrintf("Warning: not punishing manually-connected peer %s!\n", pnode->addr.ToString()); else { pnode->fDisconnect = true; if (pnode->addr.IsLocal()) @@ -2683,7 +2930,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter msg.SetVersion(pfrom->GetRecvVersion()); // Scan for message start if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->GetId()); + LogPrint(BCLog::NET, "PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->GetId()); pfrom->fDisconnect = true; return false; } @@ -2692,7 +2939,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter CMessageHeader& hdr = msg.hdr; if (!hdr.IsValid(chainparams.MessageStart())) { - LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->GetId()); + LogPrint(BCLog::NET, "PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->GetId()); return fMoreWork; } std::string strCommand = hdr.GetCommand(); @@ -2705,7 +2952,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter const uint256& hash = msg.GetMessageHash(); if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { - LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, + LogPrint(BCLog::NET, "%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, SanitizeString(strCommand), nMessageSize, HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); @@ -2728,17 +2975,17 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv - LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else if (strstr(e.what(), "size too large")) { // Allow exceptions from over-long size - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else if (strstr(e.what(), "non-canonical ReadCompactSize()")) { // Allow exceptions from non-canonical encoding - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else { @@ -2752,7 +2999,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter } if (!fRet) { - LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + LogPrint(BCLog::NET, "%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); } LOCK(cs_main); @@ -2761,6 +3008,136 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter return fMoreWork; } +void PeerLogicValidation::ConsiderEviction(CNode *pto, int64_t time_in_seconds) +{ + AssertLockHeld(cs_main); + + CNodeState &state = *State(pto->GetId()); + const CNetMsgMaker msgMaker(pto->GetSendVersion()); + + if (!state.m_chain_sync.m_protect && IsOutboundDisconnectionCandidate(pto) && state.fSyncStarted) { + // This is an outbound peer subject to disconnection if they don't + // announce a block with as much work as the current tip within + // CHAIN_SYNC_TIMEOUT + HEADERS_RESPONSE_TIME seconds (note: if + // their chain has more work than ours, we should sync to it, + // unless it's invalid, in which case we should find that out and + // disconnect from them elsewhere). + if (state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= chainActive.Tip()->nChainWork) { + if (state.m_chain_sync.m_timeout != 0) { + state.m_chain_sync.m_timeout = 0; + state.m_chain_sync.m_work_header = nullptr; + state.m_chain_sync.m_sent_getheaders = false; + } + } else if (state.m_chain_sync.m_timeout == 0 || (state.m_chain_sync.m_work_header != nullptr && state.pindexBestKnownBlock != nullptr && state.pindexBestKnownBlock->nChainWork >= state.m_chain_sync.m_work_header->nChainWork)) { + // Our best block known by this peer is behind our tip, and we're either noticing + // that for the first time, OR this peer was able to catch up to some earlier point + // where we checked against our tip. + // Either way, set a new timeout based on current tip. + state.m_chain_sync.m_timeout = time_in_seconds + CHAIN_SYNC_TIMEOUT; + state.m_chain_sync.m_work_header = chainActive.Tip(); + state.m_chain_sync.m_sent_getheaders = false; + } else if (state.m_chain_sync.m_timeout > 0 && time_in_seconds > state.m_chain_sync.m_timeout) { + // No evidence yet that our peer has synced to a chain with work equal to that + // of our tip, when we first detected it was behind. Send a single getheaders + // message to give the peer a chance to update us. + if (state.m_chain_sync.m_sent_getheaders) { + // They've run out of time to catch up! + LogPrintf("Disconnecting outbound peer %d for old chain, best known block = %s\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>"); + pto->fDisconnect = true; + } else { + assert(state.m_chain_sync.m_work_header); + LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); + state.m_chain_sync.m_sent_getheaders = true; + constexpr int64_t HEADERS_RESPONSE_TIME = 120; // 2 minutes + // Bump the timeout to allow a response, which could clear the timeout + // (if the response shows the peer has synced), reset the timeout (if + // the peer syncs to the required work but not to our tip), or result + // in disconnect (if we advance to the timeout and pindexBestKnownBlock + // has not sufficiently progressed) + state.m_chain_sync.m_timeout = time_in_seconds + HEADERS_RESPONSE_TIME; + } + } + } +} + +void PeerLogicValidation::EvictExtraOutboundPeers(int64_t time_in_seconds) +{ + // Check whether we have too many outbound peers + int extra_peers = connman->GetExtraOutboundCount(); + if (extra_peers > 0) { + // If we have more outbound peers than we target, disconnect one. + // Pick the outbound peer that least recently announced + // us a new block, with ties broken by choosing the more recent + // connection (higher node id) + NodeId worst_peer = -1; + int64_t oldest_block_announcement = std::numeric_limits<int64_t>::max(); + + LOCK(cs_main); + + connman->ForEachNode([&](CNode* pnode) { + // Ignore non-outbound peers, or nodes marked for disconnect already + if (!IsOutboundDisconnectionCandidate(pnode) || pnode->fDisconnect) return; + CNodeState *state = State(pnode->GetId()); + if (state == nullptr) return; // shouldn't be possible, but just in case + // Don't evict our protected peers + if (state->m_chain_sync.m_protect) return; + if (state->m_last_block_announcement < oldest_block_announcement || (state->m_last_block_announcement == oldest_block_announcement && pnode->GetId() > worst_peer)) { + worst_peer = pnode->GetId(); + oldest_block_announcement = state->m_last_block_announcement; + } + }); + if (worst_peer != -1) { + bool disconnected = connman->ForNode(worst_peer, [&](CNode *pnode) { + // Only disconnect a peer that has been connected to us for + // some reasonable fraction of our check-frequency, to give + // it time for new information to have arrived. + // Also don't disconnect any peer we're trying to download a + // block from. + CNodeState &state = *State(pnode->GetId()); + if (time_in_seconds - pnode->nTimeConnected > MINIMUM_CONNECT_TIME && state.nBlocksInFlight == 0) { + LogPrint(BCLog::NET, "disconnecting extra outbound peer=%d (last block announcement received at time %d)\n", pnode->GetId(), oldest_block_announcement); + pnode->fDisconnect = true; + return true; + } else { + LogPrint(BCLog::NET, "keeping outbound peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", pnode->GetId(), pnode->nTimeConnected, state.nBlocksInFlight); + return false; + } + }); + if (disconnected) { + // If we disconnected an extra peer, that means we successfully + // connected to at least one peer after the last time we + // detected a stale tip. Don't try any more extra peers until + // we next detect a stale tip, to limit the load we put on the + // network from these extra connections. + connman->SetTryNewOutboundPeer(false); + } + } + } +} + +void PeerLogicValidation::CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams) +{ + if (connman == nullptr) return; + + int64_t time_in_seconds = GetTime(); + + EvictExtraOutboundPeers(time_in_seconds); + + if (time_in_seconds > m_stale_tip_check_time) { + LOCK(cs_main); + // Check whether our tip is stale, and if so, allow using an extra + // outbound peer + if (TipMayBeStale(consensusParams)) { + LogPrintf("Potential stale tip detected, will try using extra outbound peer (last tip update: %d seconds ago)\n", time_in_seconds - g_last_tip_update); + connman->SetTryNewOutboundPeer(true); + } else if (connman->GetTryNewOutboundPeer()) { + connman->SetTryNewOutboundPeer(false); + } + m_stale_tip_check_time = time_in_seconds + STALE_CHECK_INTERVAL; + } +} + class CompareInvMempoolOrder { CTxMemPool *mp; @@ -3227,6 +3604,9 @@ bool PeerLogicValidation::SendMessages(CNode* pto, std::atomic<bool>& interruptM } } + // Check that outbound peers have reasonable chains + // GetTime() is used by this anti-DoS logic so we can test this using mocktime + ConsiderEviction(pto, GetTime()); // // Message: getdata (blocks) diff --git a/src/net_processing.h b/src/net_processing.h index 79745cdd42..4f2b8935f4 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -1,13 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_NET_PROCESSING_H #define BITCOIN_NET_PROCESSING_H -#include "net.h" -#include "validationinterface.h" +#include <net.h> +#include <validationinterface.h> +#include <consensus/params.h> /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; @@ -21,13 +22,25 @@ static const unsigned int DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN = 100; * Timeout = base + per_header * (expected number of headers) */ static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_BASE = 15 * 60 * 1000000; // 15 minutes static constexpr int64_t HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1000; // 1ms/header +/** Protect at least this many outbound peers from disconnection due to slow/ + * behind headers chain. + */ +static constexpr int32_t MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT = 4; +/** Timeout for (unprotected) outbound peers to sync to our chainwork, in seconds */ +static constexpr int64_t CHAIN_SYNC_TIMEOUT = 20 * 60; // 20 minutes +/** How frequently to check for stale tips, in seconds */ +static constexpr int64_t STALE_CHECK_INTERVAL = 10 * 60; // 10 minutes +/** How frequently to check for extra outbound peers and disconnect, in seconds */ +static constexpr int64_t EXTRA_PEER_CHECK_INTERVAL = 45; +/** Minimum time an outbound-peer-eviction candidate must be connected for, in order to evict, in seconds */ +static constexpr int64_t MINIMUM_CONNECT_TIME = 30; class PeerLogicValidation : public CValidationInterface, public NetEventsInterface { private: - CConnman* connman; + CConnman* const connman; public: - explicit PeerLogicValidation(CConnman* connman); + explicit PeerLogicValidation(CConnman* connman, CScheduler &scheduler); void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected, const std::vector<CTransactionRef>& vtxConflicted) override; void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) override; @@ -47,6 +60,13 @@ public: * @return True if there is more work to be done */ bool SendMessages(CNode* pto, std::atomic<bool>& interrupt) override; + + void ConsiderEviction(CNode *pto, int64_t time_in_seconds); + void CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams); + void EvictExtraOutboundPeers(int64_t time_in_seconds); + +private: + int64_t m_stale_tip_check_time; //! Next time to check for stale tip }; struct CNodeStateStats { diff --git a/src/netaddress.cpp b/src/netaddress.cpp index b8a261c921..81f72879f4 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -1,16 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifdef HAVE_CONFIG_H -#include "config/bitcoin-config.h" -#endif - -#include "netaddress.h" -#include "hash.h" -#include "utilstrencodings.h" -#include "tinyformat.h" +#include <netaddress.h> +#include <hash.h> +#include <utilstrencodings.h> +#include <tinyformat.h> static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; static const unsigned char pchOnionCat[] = {0xFD,0x87,0xD8,0x7E,0xEB,0x43}; diff --git a/src/netaddress.h b/src/netaddress.h index 6ca99b36b5..93bbb66491 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,11 +6,11 @@ #define BITCOIN_NETADDRESS_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "compat.h" -#include "serialize.h" +#include <compat.h> +#include <serialize.h> #include <stdint.h> #include <string> diff --git a/src/netbase.cpp b/src/netbase.cpp index 5a560bc95a..276b2f4dc2 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -1,20 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifdef HAVE_CONFIG_H -#include "config/bitcoin-config.h" -#endif - -#include "netbase.h" +#include <netbase.h> -#include "hash.h" -#include "sync.h" -#include "uint256.h" -#include "random.h" -#include "util.h" -#include "utilstrencodings.h" +#include <hash.h> +#include <sync.h> +#include <uint256.h> +#include <random.h> +#include <util.h> +#include <utilstrencodings.h> #include <atomic> @@ -291,7 +287,7 @@ struct ProxyCredentials std::string password; }; -/** Convert SOCKS5 reply to a an error message */ +/** Convert SOCKS5 reply to an error message */ std::string Socks5ErrorString(uint8_t err) { switch(err) { @@ -317,12 +313,11 @@ std::string Socks5ErrorString(uint8_t err) } /** Connect using SOCKS5 (as described in RFC1928) */ -static bool Socks5(const std::string& strDest, int port, const ProxyCredentials *auth, SOCKET& hSocket) +static bool Socks5(const std::string& strDest, int port, const ProxyCredentials *auth, const SOCKET& hSocket) { IntrRecvError recvr; LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); if (strDest.size() > 255) { - CloseSocket(hSocket); return error("Hostname too long"); } // Accepted authentication methods @@ -338,17 +333,14 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials } ssize_t ret = send(hSocket, (const char*)vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5Init.size()) { - CloseSocket(hSocket); return error("Error sending to proxy"); } uint8_t pchRet1[2]; if ((recvr = InterruptibleRecv(pchRet1, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { - CloseSocket(hSocket); LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); return false; } if (pchRet1[0] != SOCKSVersion::SOCKS5) { - CloseSocket(hSocket); return error("Proxy failed to initialize"); } if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { @@ -363,23 +355,19 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); ret = send(hSocket, (const char*)vAuth.data(), vAuth.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vAuth.size()) { - CloseSocket(hSocket); return error("Error sending authentication to proxy"); } LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); uint8_t pchRetA[2]; if ((recvr = InterruptibleRecv(pchRetA, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { - CloseSocket(hSocket); return error("Error reading proxy authentication response"); } if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { - CloseSocket(hSocket); return error("Proxy authentication unsuccessful"); } } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { // Perform no authentication } else { - CloseSocket(hSocket); return error("Proxy requested wrong authentication method %02x", pchRet1[1]); } std::vector<uint8_t> vSocks5; @@ -393,12 +381,10 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials vSocks5.push_back((port >> 0) & 0xFF); ret = send(hSocket, (const char*)vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); if (ret != (ssize_t)vSocks5.size()) { - CloseSocket(hSocket); return error("Error sending to proxy"); } uint8_t pchRet2[4]; if ((recvr = InterruptibleRecv(pchRet2, 4, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { - CloseSocket(hSocket); if (recvr == IntrRecvError::Timeout) { /* If a timeout happens here, this effectively means we timed out while connecting * to the remote node. This is very common for Tor, so do not print an @@ -409,17 +395,14 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials } } if (pchRet2[0] != SOCKSVersion::SOCKS5) { - CloseSocket(hSocket); return error("Proxy failed to accept request"); } if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { // Failures to connect to a peer that are not proxy errors - CloseSocket(hSocket); LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); return false; } if (pchRet2[2] != 0x00) { // Reserved field must be 0 - CloseSocket(hSocket); return error("Error: malformed proxy response"); } uint8_t pchRet3[256]; @@ -431,41 +414,42 @@ static bool Socks5(const std::string& strDest, int port, const ProxyCredentials { recvr = InterruptibleRecv(pchRet3, 1, SOCKS5_RECV_TIMEOUT, hSocket); if (recvr != IntrRecvError::OK) { - CloseSocket(hSocket); return error("Error reading from proxy"); } int nRecv = pchRet3[0]; recvr = InterruptibleRecv(pchRet3, nRecv, SOCKS5_RECV_TIMEOUT, hSocket); break; } - default: CloseSocket(hSocket); return error("Error: malformed proxy response"); + default: return error("Error: malformed proxy response"); } if (recvr != IntrRecvError::OK) { - CloseSocket(hSocket); return error("Error reading from proxy"); } if ((recvr = InterruptibleRecv(pchRet3, 2, SOCKS5_RECV_TIMEOUT, hSocket)) != IntrRecvError::OK) { - CloseSocket(hSocket); return error("Error reading from proxy"); } LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); return true; } -bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int nTimeout) +SOCKET CreateSocket(const CService &addrConnect) { - hSocketRet = INVALID_SOCKET; - struct sockaddr_storage sockaddr; socklen_t len = sizeof(sockaddr); if (!addrConnect.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { - LogPrintf("Cannot connect to %s: unsupported network\n", addrConnect.ToString()); - return false; + LogPrintf("Cannot create socket for %s: unsupported network\n", addrConnect.ToString()); + return INVALID_SOCKET; } SOCKET hSocket = socket(((struct sockaddr*)&sockaddr)->sa_family, SOCK_STREAM, IPPROTO_TCP); if (hSocket == INVALID_SOCKET) - return false; + return INVALID_SOCKET; + + if (!IsSelectableSocket(hSocket)) { + CloseSocket(hSocket); + LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); + return INVALID_SOCKET; + } #ifdef SO_NOSIGPIPE int set = 1; @@ -479,9 +463,23 @@ bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int // Set to non-blocking if (!SetSocketNonBlocking(hSocket, true)) { CloseSocket(hSocket); - return error("ConnectSocketDirectly: Setting socket to non-blocking failed, error %s\n", NetworkErrorString(WSAGetLastError())); + LogPrintf("ConnectSocketDirectly: Setting socket to non-blocking failed, error %s\n", NetworkErrorString(WSAGetLastError())); } + return hSocket; +} +bool ConnectSocketDirectly(const CService &addrConnect, const SOCKET& hSocket, int nTimeout) +{ + struct sockaddr_storage sockaddr; + socklen_t len = sizeof(sockaddr); + if (hSocket == INVALID_SOCKET) { + LogPrintf("Cannot connect to %s: invalid socket\n", addrConnect.ToString()); + return false; + } + if (!addrConnect.GetSockAddr((struct sockaddr*)&sockaddr, &len)) { + LogPrintf("Cannot connect to %s: unsupported network\n", addrConnect.ToString()); + return false; + } if (connect(hSocket, (struct sockaddr*)&sockaddr, len) == SOCKET_ERROR) { int nErr = WSAGetLastError(); @@ -496,13 +494,11 @@ bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int if (nRet == 0) { LogPrint(BCLog::NET, "connection to %s timeout\n", addrConnect.ToString()); - CloseSocket(hSocket); return false; } if (nRet == SOCKET_ERROR) { LogPrintf("select() for %s failed: %s\n", addrConnect.ToString(), NetworkErrorString(WSAGetLastError())); - CloseSocket(hSocket); return false; } socklen_t nRetSize = sizeof(nRet); @@ -513,13 +509,11 @@ bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int #endif { LogPrintf("getsockopt() for %s failed: %s\n", addrConnect.ToString(), NetworkErrorString(WSAGetLastError())); - CloseSocket(hSocket); return false; } if (nRet != 0) { LogPrintf("connect() to %s failed after select(): %s\n", addrConnect.ToString(), NetworkErrorString(nRet)); - CloseSocket(hSocket); return false; } } @@ -530,12 +524,9 @@ bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int #endif { LogPrintf("connect() to %s failed: %s\n", addrConnect.ToString(), NetworkErrorString(WSAGetLastError())); - CloseSocket(hSocket); return false; } } - - hSocketRet = hSocket; return true; } @@ -587,9 +578,8 @@ bool IsProxy(const CNetAddr &addr) { return false; } -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed) +bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocket, int nTimeout, bool *outProxyConnectionFailed) { - SOCKET hSocket = INVALID_SOCKET; // first connect to proxy server if (!ConnectSocketDirectly(proxy.proxy, hSocket, nTimeout)) { if (outProxyConnectionFailed) @@ -601,14 +591,14 @@ bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int ProxyCredentials random_auth; static std::atomic_int counter(0); random_auth.username = random_auth.password = strprintf("%i", counter++); - if (!Socks5(strDest, (unsigned short)port, &random_auth, hSocket)) + if (!Socks5(strDest, (unsigned short)port, &random_auth, hSocket)) { return false; + } } else { - if (!Socks5(strDest, (unsigned short)port, 0, hSocket)) + if (!Socks5(strDest, (unsigned short)port, 0, hSocket)) { return false; + } } - - hSocketRet = hSocket; return true; } bool LookupSubNet(const char* pszName, CSubNet& ret) diff --git a/src/netbase.h b/src/netbase.h index e7d7bcb375..c0921b6441 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,12 +6,12 @@ #define BITCOIN_NETBASE_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "compat.h" -#include "netaddress.h" -#include "serialize.h" +#include <compat.h> +#include <netaddress.h> +#include <serialize.h> #include <stdint.h> #include <string> @@ -51,8 +51,9 @@ bool Lookup(const char *pszName, CService& addr, int portDefault, bool fAllowLoo bool Lookup(const char *pszName, std::vector<CService>& vAddr, int portDefault, bool fAllowLookup, unsigned int nMaxSolutions); CService LookupNumeric(const char *pszName, int portDefault = 0); bool LookupSubNet(const char *pszName, CSubNet& subnet); -bool ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRet, int nTimeout); -bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed); +SOCKET CreateSocket(const CService &addrConnect); +bool ConnectSocketDirectly(const CService &addrConnect, const SOCKET& hSocketRet, int nTimeout); +bool ConnectThroughProxy(const proxyType &proxy, const std::string& strDest, int port, const SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed); /** Return readable error string for a network error code */ std::string NetworkErrorString(int err); /** Close socket and set hSocket to INVALID_SOCKET */ diff --git a/src/netmessagemaker.h b/src/netmessagemaker.h index 79b2501c5d..975be460b3 100644 --- a/src/netmessagemaker.h +++ b/src/netmessagemaker.h @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_NETMESSAGEMAKER_H #define BITCOIN_NETMESSAGEMAKER_H -#include "net.h" -#include "serialize.h" +#include <net.h> +#include <serialize.h> class CNetMsgMaker { diff --git a/src/noui.cpp b/src/noui.cpp index 4bfb75fa29..efe6f28246 100644 --- a/src/noui.cpp +++ b/src/noui.cpp @@ -1,12 +1,12 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "noui.h" +#include <noui.h> -#include "ui_interface.h" -#include "util.h" +#include <ui_interface.h> +#include <util.h> #include <cstdio> #include <stdint.h> diff --git a/src/policy/feerate.cpp b/src/policy/feerate.cpp index a089c02284..a560815491 100644 --- a/src/policy/feerate.cpp +++ b/src/policy/feerate.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "feerate.h" +#include <policy/feerate.h> -#include "tinyformat.h" +#include <tinyformat.h> const std::string CURRENCY_UNIT = "BTC"; diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 3449cdd699..eed77d5030 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POLICY_FEERATE_H #define BITCOIN_POLICY_FEERATE_H -#include "amount.h" -#include "serialize.h" +#include <amount.h> +#include <serialize.h> #include <string> diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 8056f385ab..9142f3706d 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1,18 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "policy/fees.h" -#include "policy/policy.h" +#include <policy/fees.h> +#include <policy/policy.h> -#include "amount.h" -#include "clientversion.h" -#include "primitives/transaction.h" -#include "random.h" -#include "streams.h" -#include "txmempool.h" -#include "util.h" +#include <clientversion.h> +#include <primitives/transaction.h> +#include <streams.h> +#include <txmempool.h> +#include <util.h> static constexpr double INF_FEERATE = 1e99; @@ -180,6 +178,7 @@ TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets, : buckets(defaultBuckets), bucketMap(defaultBucketMap) { decay = _decay; + assert(_scale != 0 && "_scale must be non-zero"); scale = _scale; confAvg.resize(maxPeriods); for (unsigned int i = 0; i < maxPeriods; i++) { @@ -412,12 +411,13 @@ void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets size_t maxConfirms, maxPeriods; // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor - if (nFileVersion >= 149900) { - filein >> decay; - if (decay <= 0 || decay >= 1) { - throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - } - filein >> scale; + filein >> decay; + if (decay <= 0 || decay >= 1) { + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + } + filein >> scale; + if (scale == 0) { + throw std::runtime_error("Corrupt estimates file. Scale must be non-zero"); } filein >> avg; @@ -441,20 +441,13 @@ void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets } } - if (nFileVersion >= 149900) { - filein >> failAvg; - if (maxPeriods != failAvg.size()) { - throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); - } - for (unsigned int i = 0; i < maxPeriods; i++) { - if (failAvg[i].size() != numBuckets) { - throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts"); - } - } - } else { - failAvg.resize(confAvg.size()); - for (unsigned int i = 0; i < failAvg.size(); i++) { - failAvg[i].resize(numBuckets); + filein >> failAvg; + if (maxPeriods != failAvg.size()) { + throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (failAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts"); } } @@ -503,6 +496,7 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe } } if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period + assert(scale != 0); unsigned int periodsAgo = blocksAgo / scale; for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) { failAvg[i][bucketindex]++; @@ -543,16 +537,13 @@ CBlockPolicyEstimator::CBlockPolicyEstimator() bucketMap[INF_FEERATE] = bucketIndex; assert(bucketMap.size() == buckets.size()); - feeStats = new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE); - shortStats = new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE); - longStats = new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE); + feeStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); + shortStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); + longStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); } CBlockPolicyEstimator::~CBlockPolicyEstimator() { - delete feeStats; - delete shortStats; - delete longStats; } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) @@ -563,7 +554,7 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo if (mapMemPoolTxs.count(hash)) { LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n", hash.ToString().c_str()); - return; + return; } if (txHeight != nBestSeenHeight) { @@ -685,16 +676,16 @@ CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThr double sufficientTxs = SUFFICIENT_FEETXS; switch (horizon) { case FeeEstimateHorizon::SHORT_HALFLIFE: { - stats = shortStats; + stats = shortStats.get(); sufficientTxs = SUFFICIENT_TXS_SHORT; break; } case FeeEstimateHorizon::MED_HALFLIFE: { - stats = feeStats; + stats = feeStats.get(); break; } case FeeEstimateHorizon::LONG_HALFLIFE: { - stats = longStats; + stats = longStats.get(); break; } default: { @@ -944,32 +935,9 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) unsigned int nFileBestSeenHeight; filein >> nFileBestSeenHeight; - if (nVersionThatWrote < 149900) { - // Read the old fee estimates file for temporary use, but then discard. Will start collecting data from scratch. - // decay is stored before buckets in old versions, so pre-read decay and pass into TxConfirmStats constructor - double tempDecay; - filein >> tempDecay; - if (tempDecay <= 0 || tempDecay >= 1) - throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - - std::vector<double> tempBuckets; - filein >> tempBuckets; - size_t tempNum = tempBuckets.size(); - if (tempNum <= 1 || tempNum > 1000) - throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); - - std::map<double, unsigned int> tempMap; - - std::unique_ptr<TxConfirmStats> tempFeeStats(new TxConfirmStats(tempBuckets, tempMap, MED_BLOCK_PERIODS, tempDecay, 1)); - tempFeeStats->Read(filein, nVersionThatWrote, tempNum); - // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. - - tempMap.clear(); - for (unsigned int i = 0; i < tempBuckets.size(); i++) { - tempMap[tempBuckets[i]] = i; - } - } - else { // nVersionThatWrote >= 149900 + if (nVersionRequired < 149900) { + LogPrintf("%s: incompatible old fee estimation data (non-fatal). Version: %d\n", __func__, nVersionRequired); + } else { // New format introduced in 149900 unsigned int nFileHistoricalFirst, nFileHistoricalBest; filein >> nFileHistoricalFirst >> nFileHistoricalBest; if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) { @@ -997,12 +965,9 @@ bool CBlockPolicyEstimator::Read(CAutoFile& filein) } // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap - delete feeStats; - delete shortStats; - delete longStats; - feeStats = fileFeeStats.release(); - shortStats = fileShortStats.release(); - longStats = fileLongStats.release(); + feeStats = std::move(fileFeeStats); + shortStats = std::move(fileShortStats); + longStats = std::move(fileLongStats); nBestSeenHeight = nFileBestSeenHeight; historicalFirst = nFileHistoricalFirst; diff --git a/src/policy/fees.h b/src/policy/fees.h index 6edaf28714..96a842b7a1 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -1,15 +1,15 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POLICYESTIMATOR_H #define BITCOIN_POLICYESTIMATOR_H -#include "amount.h" -#include "feerate.h" -#include "uint256.h" -#include "random.h" -#include "sync.h" +#include <amount.h> +#include <policy/feerate.h> +#include <uint256.h> +#include <random.h> +#include <sync.h> #include <map> #include <string> @@ -245,9 +245,9 @@ private: std::map<uint256, TxStatsInfo> mapMemPoolTxs; /** Classes to track historical data on transaction confirmations */ - TxConfirmStats* feeStats; - TxConfirmStats* shortStats; - TxConfirmStats* longStats; + std::unique_ptr<TxConfirmStats> feeStats; + std::unique_ptr<TxConfirmStats> shortStats; + std::unique_ptr<TxConfirmStats> longStats; unsigned int trackedTxs; unsigned int untrackedTxs; diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index b2fb284508..bff58932b4 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -1,18 +1,18 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // NOTE: This file is intended to be customised by the end user, and includes only local node policy logic -#include "policy/policy.h" +#include <policy/policy.h> -#include "consensus/validation.h" -#include "validation.h" -#include "coins.h" -#include "tinyformat.h" -#include "util.h" -#include "utilstrencodings.h" +#include <consensus/validation.h> +#include <validation.h> +#include <coins.h> +#include <tinyformat.h> +#include <util.h> +#include <utilstrencodings.h> CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) diff --git a/src/policy/policy.h b/src/policy/policy.h index ef71dd73bc..3d96406bbc 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -1,15 +1,15 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POLICY_POLICY_H #define BITCOIN_POLICY_POLICY_H -#include "consensus/consensus.h" -#include "feerate.h" -#include "script/interpreter.h" -#include "script/standard.h" +#include <consensus/consensus.h> +#include <policy/feerate.h> +#include <script/interpreter.h> +#include <script/standard.h> #include <string> @@ -49,28 +49,28 @@ static const unsigned int DUST_RELAY_TX_FEE = 3000; * with. However scripts violating these flags may still be present in valid * blocks and we must accept those blocks. */ -static const unsigned int STANDARD_SCRIPT_VERIFY_FLAGS = MANDATORY_SCRIPT_VERIFY_FLAGS | - SCRIPT_VERIFY_DERSIG | - SCRIPT_VERIFY_STRICTENC | - SCRIPT_VERIFY_MINIMALDATA | - SCRIPT_VERIFY_NULLDUMMY | - SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS | - SCRIPT_VERIFY_CLEANSTACK | - SCRIPT_VERIFY_MINIMALIF | - SCRIPT_VERIFY_NULLFAIL | - SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY | - SCRIPT_VERIFY_CHECKSEQUENCEVERIFY | - SCRIPT_VERIFY_LOW_S | - SCRIPT_VERIFY_WITNESS | - SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM | - SCRIPT_VERIFY_WITNESS_PUBKEYTYPE; +static constexpr unsigned int STANDARD_SCRIPT_VERIFY_FLAGS = MANDATORY_SCRIPT_VERIFY_FLAGS | + SCRIPT_VERIFY_DERSIG | + SCRIPT_VERIFY_STRICTENC | + SCRIPT_VERIFY_MINIMALDATA | + SCRIPT_VERIFY_NULLDUMMY | + SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS | + SCRIPT_VERIFY_CLEANSTACK | + SCRIPT_VERIFY_MINIMALIF | + SCRIPT_VERIFY_NULLFAIL | + SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY | + SCRIPT_VERIFY_CHECKSEQUENCEVERIFY | + SCRIPT_VERIFY_LOW_S | + SCRIPT_VERIFY_WITNESS | + SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM | + SCRIPT_VERIFY_WITNESS_PUBKEYTYPE; /** For convenience, standard but not mandatory verify flags. */ -static const unsigned int STANDARD_NOT_MANDATORY_VERIFY_FLAGS = STANDARD_SCRIPT_VERIFY_FLAGS & ~MANDATORY_SCRIPT_VERIFY_FLAGS; +static constexpr unsigned int STANDARD_NOT_MANDATORY_VERIFY_FLAGS = STANDARD_SCRIPT_VERIFY_FLAGS & ~MANDATORY_SCRIPT_VERIFY_FLAGS; /** Used as the flags parameter to sequence and nLocktime checks in non-consensus code. */ -static const unsigned int STANDARD_LOCKTIME_VERIFY_FLAGS = LOCKTIME_VERIFY_SEQUENCE | - LOCKTIME_MEDIAN_TIME_PAST; +static constexpr unsigned int STANDARD_LOCKTIME_VERIFY_FLAGS = LOCKTIME_VERIFY_SEQUENCE | + LOCKTIME_MEDIAN_TIME_PAST; CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee); diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 755ef83c9a..c5a1741608 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "policy/rbf.h" +#include <policy/rbf.h> bool SignalsOptInRBF(const CTransaction &tx) { diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 22c73f3319..72f51b0f03 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -1,11 +1,11 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POLICY_RBF_H #define BITCOIN_POLICY_RBF_H -#include "txmempool.h" +#include <txmempool.h> static const uint32_t MAX_BIP125_RBF_SEQUENCE = 0xfffffffd; diff --git a/src/pow.cpp b/src/pow.cpp index 7d87c6bbb7..929ea26a02 100644 --- a/src/pow.cpp +++ b/src/pow.cpp @@ -1,14 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "pow.h" +#include <pow.h> -#include "arith_uint256.h" -#include "chain.h" -#include "primitives/block.h" -#include "uint256.h" +#include <arith_uint256.h> +#include <chain.h> +#include <primitives/block.h> +#include <uint256.h> unsigned int GetNextWorkRequired(const CBlockIndex* pindexLast, const CBlockHeader *pblock, const Consensus::Params& params) { @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_POW_H #define BITCOIN_POW_H -#include "consensus/params.h" +#include <consensus/params.h> #include <stdint.h> diff --git a/src/prevector.h b/src/prevector.h index eb29b3ae7e..f8d6a09145 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 3774ac3e4b..21f880bccb 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -1,14 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "primitives/block.h" +#include <primitives/block.h> -#include "hash.h" -#include "tinyformat.h" -#include "utilstrencodings.h" -#include "crypto/common.h" +#include <hash.h> +#include <tinyformat.h> +#include <utilstrencodings.h> +#include <crypto/common.h> uint256 CBlockHeader::GetHash() const { diff --git a/src/primitives/block.h b/src/primitives/block.h index 292df40896..612a9fa203 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -1,14 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_PRIMITIVES_BLOCK_H #define BITCOIN_PRIMITIVES_BLOCK_H -#include "primitives/transaction.h" -#include "serialize.h" -#include "uint256.h" +#include <primitives/transaction.h> +#include <serialize.h> +#include <uint256.h> /** Nodes collect new transactions into a block, hash them into a hash tree, * and scan through nonce values to make the block's hash satisfy proof-of-work diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index e0a106adb9..6f463cabf5 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "primitives/transaction.h" +#include <primitives/transaction.h> -#include "hash.h" -#include "tinyformat.h" -#include "utilstrencodings.h" +#include <hash.h> +#include <tinyformat.h> +#include <utilstrencodings.h> std::string COutPoint::ToString() const { diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 18d524e23d..cd348fdbe4 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,10 +7,10 @@ #define BITCOIN_PRIMITIVES_TRANSACTION_H #include <stdint.h> -#include "amount.h" -#include "script/script.h" -#include "serialize.h" -#include "uint256.h" +#include <amount.h> +#include <script/script.h> +#include <serialize.h> +#include <uint256.h> static const int SERIALIZE_TRANSACTION_NO_WITNESS = 0x40000000; diff --git a/src/protocol.cpp b/src/protocol.cpp index da87e40091..c412ad9ffe 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "protocol.h" +#include <protocol.h> -#include "util.h" -#include "utilstrencodings.h" +#include <util.h> +#include <utilstrencodings.h> #ifndef WIN32 # include <arpa/inet.h> diff --git a/src/protocol.h b/src/protocol.h index 67e01d9606..452b5b28ce 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,10 +10,10 @@ #ifndef BITCOIN_PROTOCOL_H #define BITCOIN_PROTOCOL_H -#include "netaddress.h" -#include "serialize.h" -#include "uint256.h" -#include "version.h" +#include <netaddress.h> +#include <serialize.h> +#include <uint256.h> +#include <version.h> #include <stdint.h> #include <string> @@ -27,16 +27,13 @@ class CMessageHeader { public: - enum { - MESSAGE_START_SIZE = 4, - COMMAND_SIZE = 12, - MESSAGE_SIZE_SIZE = 4, - CHECKSUM_SIZE = 4, - - MESSAGE_SIZE_OFFSET = MESSAGE_START_SIZE + COMMAND_SIZE, - CHECKSUM_OFFSET = MESSAGE_SIZE_OFFSET + MESSAGE_SIZE_SIZE, - HEADER_SIZE = MESSAGE_START_SIZE + COMMAND_SIZE + MESSAGE_SIZE_SIZE + CHECKSUM_SIZE - }; + static constexpr size_t MESSAGE_START_SIZE = 4; + static constexpr size_t COMMAND_SIZE = 12; + static constexpr size_t MESSAGE_SIZE_SIZE = 4; + static constexpr size_t CHECKSUM_SIZE = 4; + static constexpr size_t MESSAGE_SIZE_OFFSET = MESSAGE_START_SIZE + COMMAND_SIZE; + static constexpr size_t CHECKSUM_OFFSET = MESSAGE_SIZE_OFFSET + MESSAGE_SIZE_SIZE; + static constexpr size_t HEADER_SIZE = MESSAGE_START_SIZE + COMMAND_SIZE + MESSAGE_SIZE_SIZE + CHECKSUM_SIZE; typedef unsigned char MessageStartChars[MESSAGE_START_SIZE]; explicit CMessageHeader(const MessageStartChars& pchMessageStartIn); @@ -249,9 +246,8 @@ const std::vector<std::string> &getAllNetMessageTypes(); enum ServiceFlags : uint64_t { // Nothing NODE_NONE = 0, - // NODE_NETWORK means that the node is capable of serving the block chain. It is currently - // set by all Bitcoin Core nodes, and is unset by SPV clients or other peers that just want - // network services but don't provide them. + // NODE_NETWORK means that the node is capable of serving the complete block chain. It is currently + // set by all Bitcoin Core non pruned nodes, and is unset by SPV clients or other light clients. NODE_NETWORK = (1 << 0), // NODE_GETUTXO means the node is capable of responding to the getutxo protocol request. // Bitcoin Core does not support this but a patch set called Bitcoin XT does. @@ -267,6 +263,10 @@ enum ServiceFlags : uint64_t { // NODE_XTHIN means the node supports Xtreme Thinblocks // If this is turned off then the node will not service nor make xthin requests NODE_XTHIN = (1 << 4), + // NODE_NETWORK_LIMITED means the same as NODE_NETWORK with the limitation of only + // serving the last 288 (2 day) blocks + // See BIP159 for details on how this is implemented. + NODE_NETWORK_LIMITED = (1 << 10), // Bits 24-31 are reserved for temporary experiments. Just pick a bit that // isn't getting used, or one not being used much, and notify the @@ -277,6 +277,43 @@ enum ServiceFlags : uint64_t { // BIP process. }; +/** + * Gets the set of service flags which are "desirable" for a given peer. + * + * These are the flags which are required for a peer to support for them + * to be "interesting" to us, ie for us to wish to use one of our few + * outbound connection slots for or for us to wish to prioritize keeping + * their connection around. + * + * Relevant service flags may be peer- and state-specific in that the + * version of the peer may determine which flags are required (eg in the + * case of NODE_NETWORK_LIMITED where we seek out NODE_NETWORK peers + * unless they set NODE_NETWORK_LIMITED and we are out of IBD, in which + * case NODE_NETWORK_LIMITED suffices). + * + * Thus, generally, avoid calling with peerServices == NODE_NONE. + */ +static ServiceFlags GetDesirableServiceFlags(ServiceFlags services) { + return ServiceFlags(NODE_NETWORK | NODE_WITNESS); +} + +/** + * A shortcut for (services & GetDesirableServiceFlags(services)) + * == GetDesirableServiceFlags(services), ie determines whether the given + * set of service flags are sufficient for a peer to be "relevant". + */ +static inline bool HasAllDesirableServiceFlags(ServiceFlags services) { + return !(GetDesirableServiceFlags(services) & (~services)); +} + +/** + * Checks if a peer with the given service flags may be capable of having a + * robust address-storage DB. Currently an alias for checking NODE_NETWORK. + */ +static inline bool MayHaveUsefulAddressDB(ServiceFlags services) { + return services & NODE_NETWORK; +} + /** A CService with information about it as peer */ class CAddress : public CService { diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 2da7be783f..6e601a6f13 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -1,8 +1,9 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "pubkey.h" +#include <pubkey.h> #include <secp256k1.h> #include <secp256k1_recovery.h> @@ -46,7 +47,7 @@ static int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1 lenbyte = input[pos++]; if (lenbyte & 0x80) { lenbyte -= 0x80; - if (pos + lenbyte > inputlen) { + if (lenbyte > inputlen - pos) { return 0; } pos += lenbyte; @@ -65,14 +66,15 @@ static int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1 lenbyte = input[pos++]; if (lenbyte & 0x80) { lenbyte -= 0x80; - if (pos + lenbyte > inputlen) { + if (lenbyte > inputlen - pos) { return 0; } while (lenbyte > 0 && input[pos] == 0) { pos++; lenbyte--; } - if (lenbyte >= sizeof(size_t)) { + static_assert(sizeof(size_t) >= 4, "size_t too small"); + if (lenbyte >= 4) { return 0; } rlen = 0; @@ -103,14 +105,15 @@ static int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1 lenbyte = input[pos++]; if (lenbyte & 0x80) { lenbyte -= 0x80; - if (pos + lenbyte > inputlen) { + if (lenbyte > inputlen - pos) { return 0; } while (lenbyte > 0 && input[pos] == 0) { pos++; lenbyte--; } - if (lenbyte >= sizeof(size_t)) { + static_assert(sizeof(size_t) >= 4, "size_t too small"); + if (lenbyte >= 4) { return 0; } slen = 0; @@ -126,7 +129,6 @@ static int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1 return 0; } spos = pos; - pos += slen; /* Ignore leading zeroes in R */ while (rlen > 0 && input[rpos] == 0) { @@ -182,7 +184,7 @@ bool CPubKey::Verify(const uint256 &hash, const std::vector<unsigned char>& vchS } bool CPubKey::RecoverCompact(const uint256 &hash, const std::vector<unsigned char>& vchSig) { - if (vchSig.size() != 65) + if (vchSig.size() != COMPACT_SIGNATURE_SIZE) return false; int recid = (vchSig[0] - 27) & 3; bool fComp = ((vchSig[0] - 27) & 4) != 0; @@ -194,8 +196,8 @@ bool CPubKey::RecoverCompact(const uint256 &hash, const std::vector<unsigned cha if (!secp256k1_ecdsa_recover(secp256k1_context_verify, &pubkey, &sig, hash.begin())) { return false; } - unsigned char pub[65]; - size_t publen = 65; + unsigned char pub[PUBLIC_KEY_SIZE]; + size_t publen = PUBLIC_KEY_SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, fComp ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED); Set(pub, pub + publen); return true; @@ -215,8 +217,8 @@ bool CPubKey::Decompress() { if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, &(*this)[0], size())) { return false; } - unsigned char pub[65]; - size_t publen = 65; + unsigned char pub[PUBLIC_KEY_SIZE]; + size_t publen = PUBLIC_KEY_SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, SECP256K1_EC_UNCOMPRESSED); Set(pub, pub + publen); return true; @@ -225,7 +227,7 @@ bool CPubKey::Decompress() { bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const { assert(IsValid()); assert((nChild >> 31) == 0); - assert(begin() + 33 == end()); + assert(size() == COMPRESSED_PUBLIC_KEY_SIZE); unsigned char out[64]; BIP32Hash(cc, nChild, *begin(), begin()+1, out); memcpy(ccChild.begin(), out+32, 32); @@ -236,8 +238,8 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi if (!secp256k1_ec_pubkey_tweak_add(secp256k1_context_verify, &pubkey, out)) { return false; } - unsigned char pub[33]; - size_t publen = 33; + unsigned char pub[COMPRESSED_PUBLIC_KEY_SIZE]; + size_t publen = COMPRESSED_PUBLIC_KEY_SIZE; secp256k1_ec_pubkey_serialize(secp256k1_context_verify, pub, &publen, &pubkey, SECP256K1_EC_COMPRESSED); pubkeyChild.Set(pub, pub + publen); return true; @@ -249,8 +251,8 @@ void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; memcpy(code+9, chaincode.begin(), 32); - assert(pubkey.size() == 33); - memcpy(code+41, pubkey.begin(), 33); + assert(pubkey.size() == CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); + memcpy(code+41, pubkey.begin(), CPubKey::COMPRESSED_PUBLIC_KEY_SIZE); } void CExtPubKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { diff --git a/src/pubkey.h b/src/pubkey.h index 65738d8fe2..59bf56395c 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -1,28 +1,19 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_PUBKEY_H #define BITCOIN_PUBKEY_H -#include "hash.h" -#include "serialize.h" -#include "uint256.h" +#include <hash.h> +#include <serialize.h> +#include <uint256.h> #include <stdexcept> #include <vector> -/** - * secp256k1: - * const unsigned int PRIVATE_KEY_SIZE = 279; - * const unsigned int PUBLIC_KEY_SIZE = 65; - * const unsigned int SIGNATURE_SIZE = 72; - * - * see www.keylength.com - * script supports up to 75 for single byte push - */ - const unsigned int BIP32_EXTKEY_SIZE = 74; /** A reference to a CKey: the Hash160 of its serialized public key */ @@ -38,21 +29,37 @@ typedef uint256 ChainCode; /** An encapsulated public key. */ class CPubKey { +public: + /** + * secp256k1: + */ + static const unsigned int PUBLIC_KEY_SIZE = 65; + static const unsigned int COMPRESSED_PUBLIC_KEY_SIZE = 33; + static const unsigned int SIGNATURE_SIZE = 72; + static const unsigned int COMPACT_SIGNATURE_SIZE = 65; + /** + * see www.keylength.com + * script supports up to 75 for single byte push + */ + static_assert( + PUBLIC_KEY_SIZE >= COMPRESSED_PUBLIC_KEY_SIZE, + "COMPRESSED_PUBLIC_KEY_SIZE is larger than PUBLIC_KEY_SIZE"); + private: /** * Just store the serialized data. * Its length can very cheaply be computed from the first byte. */ - unsigned char vch[65]; + unsigned char vch[PUBLIC_KEY_SIZE]; //! Compute the length of a pubkey with a given first byte. unsigned int static GetLen(unsigned char chHeader) { if (chHeader == 2 || chHeader == 3) - return 33; + return COMPRESSED_PUBLIC_KEY_SIZE; if (chHeader == 4 || chHeader == 6 || chHeader == 7) - return 65; + return PUBLIC_KEY_SIZE; return 0; } @@ -127,7 +134,7 @@ public: void Unserialize(Stream& s) { unsigned int len = ::ReadCompactSize(s); - if (len <= 65) { + if (len <= PUBLIC_KEY_SIZE) { s.read((char*)vch, len); } else { // invalid pubkey, skip available data @@ -166,7 +173,7 @@ public: //! Check whether this is a compressed public key. bool IsCompressed() const { - return size() == 33; + return size() == COMPRESSED_PUBLIC_KEY_SIZE; } /** diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index f295bd4689..517aa49e2b 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -1,20 +1,20 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "addressbookpage.h" -#include "ui_addressbookpage.h" +#include <qt/addressbookpage.h> +#include <qt/forms/ui_addressbookpage.h> -#include "addresstablemodel.h" -#include "bitcoingui.h" -#include "csvmodelwriter.h" -#include "editaddressdialog.h" -#include "guiutil.h" -#include "platformstyle.h" +#include <qt/addresstablemodel.h> +#include <qt/bitcoingui.h> +#include <qt/csvmodelwriter.h> +#include <qt/editaddressdialog.h> +#include <qt/guiutil.h> +#include <qt/platformstyle.h> #include <QIcon> #include <QMenu> diff --git a/src/qt/addressbookpage.h b/src/qt/addressbookpage.h index 3c00fd0809..54a43478d1 100644 --- a/src/qt/addressbookpage.h +++ b/src/qt/addressbookpage.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 0eb7ec4306..a2521a1e9e 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addresstablemodel.h" +#include <qt/addresstablemodel.h> -#include "guiutil.h" -#include "walletmodel.h" +#include <qt/guiutil.h> +#include <qt/walletmodel.h> -#include "base58.h" -#include "wallet/wallet.h" +#include <base58.h> +#include <wallet/wallet.h> #include <QFont> @@ -384,7 +384,8 @@ QString AddressTableModel::addRow(const QString &type, const QString &label, con return QString(); } } - strAddress = EncodeDestination(newKey.GetID()); + wallet->LearnRelatedScripts(newKey, g_address_type); + strAddress = EncodeDestination(GetDestinationForKey(newKey, g_address_type)); } else { diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index e9f5c77a5b..0270af98b7 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -1,18 +1,18 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "askpassphrasedialog.h" -#include "ui_askpassphrasedialog.h" +#include <qt/askpassphrasedialog.h> +#include <qt/forms/ui_askpassphrasedialog.h> -#include "guiconstants.h" -#include "walletmodel.h" +#include <qt/guiconstants.h> +#include <qt/walletmodel.h> -#include "support/allocators/secure.h" +#include <support/allocators/secure.h> #include <QKeyEvent> #include <QMessageBox> @@ -70,6 +70,7 @@ AskPassphraseDialog::AskPassphraseDialog(Mode _mode, QWidget *parent) : break; } textChanged(); + connect(ui->toggleShowPasswordButton, SIGNAL(toggled(bool)), this, SLOT(toggleShowPassword(bool))); connect(ui->passEdit1, SIGNAL(textChanged(QString)), this, SLOT(textChanged())); connect(ui->passEdit2, SIGNAL(textChanged(QString)), this, SLOT(textChanged())); connect(ui->passEdit3, SIGNAL(textChanged(QString)), this, SLOT(textChanged())); @@ -234,6 +235,15 @@ bool AskPassphraseDialog::event(QEvent *event) return QWidget::event(event); } +void AskPassphraseDialog::toggleShowPassword(bool show) +{ + ui->toggleShowPasswordButton->setDown(show); + const auto mode = show ? QLineEdit::Normal : QLineEdit::Password; + ui->passEdit1->setEchoMode(mode); + ui->passEdit2->setEchoMode(mode); + ui->passEdit3->setEchoMode(mode); +} + bool AskPassphraseDialog::eventFilter(QObject *object, QEvent *event) { /* Detect Caps Lock. diff --git a/src/qt/askpassphrasedialog.h b/src/qt/askpassphrasedialog.h index 34bf7ccb31..7ec8bf0a15 100644 --- a/src/qt/askpassphrasedialog.h +++ b/src/qt/askpassphrasedialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -43,6 +43,7 @@ private: private Q_SLOTS: void textChanged(); void secureClearPassFields(); + void toggleShowPassword(bool); protected: bool event(QEvent *event); diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index f8a99506c1..c89c90e118 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bantablemodel.h" +#include <qt/bantablemodel.h> -#include "clientmodel.h" -#include "guiconstants.h" -#include "guiutil.h" +#include <qt/clientmodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> -#include "sync.h" -#include "utiltime.h" +#include <sync.h> +#include <utiltime.h> #include <QDebug> #include <QList> @@ -55,11 +55,11 @@ public: #if QT_VERSION >= 0x040700 cachedBanlist.reserve(banMap.size()); #endif - for (banmap_t::iterator it = banMap.begin(); it != banMap.end(); it++) + for (const auto& entry : banMap) { CCombinedBan banEntry; - banEntry.subnet = (*it).first; - banEntry.banEntry = (*it).second; + banEntry.subnet = entry.first; + banEntry.banEntry = entry.second; cachedBanlist.append(banEntry); } diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index 062cfdc931..a54f8793d0 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_BANTABLEMODEL_H #define BITCOIN_QT_BANTABLEMODEL_H -#include "net.h" +#include <net.h> #include <QAbstractTableModel> #include <QStringList> diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 3fd58a2f9a..0666dcb9a3 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -1,40 +1,40 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "bitcoingui.h" - -#include "chainparams.h" -#include "clientmodel.h" -#include "fs.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "intro.h" -#include "networkstyle.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "splashscreen.h" -#include "utilitydialog.h" -#include "winshutdownmonitor.h" +#include <qt/bitcoingui.h> + +#include <chainparams.h> +#include <qt/clientmodel.h> +#include <fs.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/intro.h> +#include <qt/networkstyle.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/splashscreen.h> +#include <qt/utilitydialog.h> +#include <qt/winshutdownmonitor.h> #ifdef ENABLE_WALLET -#include "paymentserver.h" -#include "walletmodel.h" +#include <qt/paymentserver.h> +#include <qt/walletmodel.h> #endif -#include "init.h" -#include "rpc/server.h" -#include "scheduler.h" -#include "ui_interface.h" -#include "util.h" -#include "warnings.h" +#include <init.h> +#include <rpc/server.h> +#include <scheduler.h> +#include <ui_interface.h> +#include <util.h> +#include <warnings.h> #ifdef ENABLE_WALLET -#include "wallet/wallet.h" +#include <wallet/wallet.h> #endif #include <stdint.h> @@ -261,7 +261,7 @@ private: void startThread(); }; -#include "bitcoin.moc" +#include <qt/bitcoin.moc> BitcoinCore::BitcoinCore(): QObject() diff --git a/src/qt/bitcoinaddressvalidator.cpp b/src/qt/bitcoinaddressvalidator.cpp index 362a71f04d..395ab447d2 100644 --- a/src/qt/bitcoinaddressvalidator.cpp +++ b/src/qt/bitcoinaddressvalidator.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bitcoinaddressvalidator.h" +#include <qt/bitcoinaddressvalidator.h> -#include "base58.h" +#include <base58.h> /* Base58 characters are: "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" diff --git a/src/qt/bitcoinamountfield.cpp b/src/qt/bitcoinamountfield.cpp index 73eb35a54e..e8307ff125 100644 --- a/src/qt/bitcoinamountfield.cpp +++ b/src/qt/bitcoinamountfield.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bitcoinamountfield.h" +#include <qt/bitcoinamountfield.h> -#include "bitcoinunits.h" -#include "guiconstants.h" -#include "qvaluecombobox.h" +#include <qt/bitcoinunits.h> +#include <qt/guiconstants.h> +#include <qt/qvaluecombobox.h> #include <QApplication> #include <QAbstractSpinBox> @@ -188,7 +188,7 @@ Q_SIGNALS: void valueChanged(); }; -#include "bitcoinamountfield.moc" +#include <qt/bitcoinamountfield.moc> BitcoinAmountField::BitcoinAmountField(QWidget *parent) : QWidget(parent), diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index 2f03a3d171..8e2cceeb5e 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_BITCOINAMOUNTFIELD_H #define BITCOIN_QT_BITCOINAMOUNTFIELD_H -#include "amount.h" +#include <amount.h> #include <QWidget> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index dc55141900..afd90a3bc6 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -1,40 +1,36 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" -#endif - -#include "bitcoingui.h" - -#include "bitcoinunits.h" -#include "clientmodel.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "modaloverlay.h" -#include "networkstyle.h" -#include "notificator.h" -#include "openuridialog.h" -#include "optionsdialog.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "rpcconsole.h" -#include "utilitydialog.h" +#include <qt/bitcoingui.h> + +#include <qt/bitcoinunits.h> +#include <qt/clientmodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/modaloverlay.h> +#include <qt/networkstyle.h> +#include <qt/notificator.h> +#include <qt/openuridialog.h> +#include <qt/optionsdialog.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/rpcconsole.h> +#include <qt/utilitydialog.h> #ifdef ENABLE_WALLET -#include "walletframe.h" -#include "walletmodel.h" +#include <qt/walletframe.h> +#include <qt/walletmodel.h> #endif // ENABLE_WALLET #ifdef Q_OS_MAC -#include "macdockiconhandler.h" +#include <qt/macdockiconhandler.h> #endif -#include "chainparams.h" -#include "init.h" -#include "ui_interface.h" -#include "util.h" +#include <chainparams.h> +#include <init.h> +#include <ui_interface.h> +#include <util.h> #include <iostream> @@ -1210,7 +1206,7 @@ UnitDisplayStatusBarControl::UnitDisplayStatusBarControl(const PlatformStyle *pl const QFontMetrics fm(font()); for (const BitcoinUnits::Unit unit : units) { - max_width = qMax(max_width, fm.width(BitcoinUnits::name(unit))); + max_width = qMax(max_width, fm.width(BitcoinUnits::longName(unit))); } setMinimumSize(max_width, 0); setAlignment(Qt::AlignRight | Qt::AlignVCenter); @@ -1229,7 +1225,7 @@ void UnitDisplayStatusBarControl::createContextMenu() menu = new QMenu(this); for (BitcoinUnits::Unit u : BitcoinUnits::availableUnits()) { - QAction *menuAction = new QAction(QString(BitcoinUnits::name(u)), this); + QAction *menuAction = new QAction(QString(BitcoinUnits::longName(u)), this); menuAction->setData(QVariant(u)); menu->addAction(menuAction); } @@ -1254,7 +1250,7 @@ void UnitDisplayStatusBarControl::setOptionsModel(OptionsModel *_optionsModel) /** When Display Units are changed on OptionsModel it will refresh the display text of the control on the status bar */ void UnitDisplayStatusBarControl::updateDisplayUnit(int newUnits) { - setText(BitcoinUnits::name(newUnits)); + setText(BitcoinUnits::longName(newUnits)); } /** Shows context menu with Display Unit options by the mouse coordinates */ diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index aa45ea1f0a..ddb7ecb76a 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,10 +6,10 @@ #define BITCOIN_QT_BITCOINGUI_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "amount.h" +#include <amount.h> #include <QLabel> #include <QMainWindow> diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index 4da414a4e3..9df05d2a13 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bitcoinunits.h" +#include <qt/bitcoinunits.h> -#include "primitives/transaction.h" +#include <primitives/transaction.h> #include <QStringList> @@ -36,24 +36,33 @@ bool BitcoinUnits::valid(int unit) } } -QString BitcoinUnits::name(int unit) +QString BitcoinUnits::longName(int unit) { switch(unit) { case BTC: return QString("BTC"); case mBTC: return QString("mBTC"); - case uBTC: return QString::fromUtf8("μBTC"); + case uBTC: return QString::fromUtf8("µBTC (bits)"); default: return QString("???"); } } +QString BitcoinUnits::shortName(int unit) +{ + switch(unit) + { + case uBTC: return QString::fromUtf8("bits"); + default: return longName(unit); + } +} + QString BitcoinUnits::description(int unit) { switch(unit) { case BTC: return QString("Bitcoins"); case mBTC: return QString("Milli-Bitcoins (1 / 1" THIN_SP_UTF8 "000)"); - case uBTC: return QString("Micro-Bitcoins (1 / 1" THIN_SP_UTF8 "000" THIN_SP_UTF8 "000)"); + case uBTC: return QString("Micro-Bitcoins (bits) (1 / 1" THIN_SP_UTF8 "000" THIN_SP_UTF8 "000)"); default: return QString("???"); } } @@ -121,7 +130,7 @@ QString BitcoinUnits::format(int unit, const CAmount& nIn, bool fPlus, Separator QString BitcoinUnits::formatWithUnit(int unit, const CAmount& amount, bool plussign, SeparatorStyle separators) { - return format(unit, amount, plussign, separators) + QString(" ") + name(unit); + return format(unit, amount, plussign, separators) + QString(" ") + shortName(unit); } QString BitcoinUnits::formatHtmlWithUnit(int unit, const CAmount& amount, bool plussign, SeparatorStyle separators) @@ -176,7 +185,7 @@ QString BitcoinUnits::getAmountColumnTitle(int unit) QString amountTitle = QObject::tr("Amount"); if (BitcoinUnits::valid(unit)) { - amountTitle += " ("+BitcoinUnits::name(unit) + ")"; + amountTitle += " ("+BitcoinUnits::shortName(unit) + ")"; } return amountTitle; } @@ -197,7 +206,7 @@ QVariant BitcoinUnits::data(const QModelIndex &index, int role) const { case Qt::EditRole: case Qt::DisplayRole: - return QVariant(name(unit)); + return QVariant(longName(unit)); case Qt::ToolTipRole: return QVariant(description(unit)); case UnitRole: diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index 6ef37de380..310f651815 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_BITCOINUNITS_H #define BITCOIN_QT_BITCOINUNITS_H -#include "amount.h" +#include <amount.h> #include <QAbstractListModel> #include <QString> @@ -76,8 +76,10 @@ public: static QList<Unit> availableUnits(); //! Is unit ID valid? static bool valid(int unit); + //! Long name + static QString longName(int unit); //! Short name - static QString name(int unit); + static QString shortName(int unit); //! Longer description static QString description(int unit); //! Number of Satoshis (1e-8) per unit diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 52ce11cefd..3642e5d4d4 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -1,24 +1,24 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "clientmodel.h" - -#include "bantablemodel.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "peertablemodel.h" - -#include "chain.h" -#include "chainparams.h" -#include "checkpoints.h" -#include "clientversion.h" -#include "validation.h" -#include "net.h" -#include "txmempool.h" -#include "ui_interface.h" -#include "util.h" -#include "warnings.h" +#include <qt/clientmodel.h> + +#include <qt/bantablemodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/peertablemodel.h> + +#include <chain.h> +#include <chainparams.h> +#include <checkpoints.h> +#include <clientversion.h> +#include <validation.h> +#include <net.h> +#include <txmempool.h> +#include <ui_interface.h> +#include <util.h> +#include <warnings.h> #include <stdint.h> diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 6447cae1bb..99ec2365a9 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 3ca43eae22..ea4d6a4816 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -1,25 +1,25 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "coincontroldialog.h" -#include "ui_coincontroldialog.h" - -#include "addresstablemodel.h" -#include "bitcoinunits.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "txmempool.h" -#include "walletmodel.h" - -#include "wallet/coincontrol.h" -#include "init.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "validation.h" // For mempool -#include "wallet/fees.h" -#include "wallet/wallet.h" +#include <qt/coincontroldialog.h> +#include <qt/forms/ui_coincontroldialog.h> + +#include <qt/addresstablemodel.h> +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <txmempool.h> +#include <qt/walletmodel.h> + +#include <wallet/coincontrol.h> +#include <init.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <validation.h> // For mempool +#include <wallet/fees.h> +#include <wallet/wallet.h> #include <QApplication> #include <QCheckBox> @@ -28,9 +28,7 @@ #include <QFlags> #include <QIcon> #include <QSettings> -#include <QString> #include <QTreeWidget> -#include <QTreeWidgetItem> QList<CAmount> CoinControlDialog::payAmounts; CCoinControl* CoinControlDialog::coinControl = new CCoinControl(); @@ -582,7 +580,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) QString toolTipDust = tr("This label turns red if any recipient receives an amount smaller than the current dust threshold."); // how many satoshis the estimated fee can vary per byte we guess wrong - double dFeeVary = (double)nPayFee / nBytes; + double dFeeVary = (nBytes != 0) ? (double)nPayFee / nBytes : 0; QString toolTip4 = tr("Can vary +/- %1 satoshi(s) per input.").arg(dFeeVary); diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h index 4949c91771..355ed5f553 100644 --- a/src/qt/coincontroldialog.h +++ b/src/qt/coincontroldialog.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_COINCONTROLDIALOG_H #define BITCOIN_QT_COINCONTROLDIALOG_H -#include "amount.h" +#include <amount.h> #include <QAbstractButton> #include <QAction> diff --git a/src/qt/coincontroltreewidget.cpp b/src/qt/coincontroltreewidget.cpp index 88510b6168..82db28d586 100644 --- a/src/qt/coincontroltreewidget.cpp +++ b/src/qt/coincontroltreewidget.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "coincontroltreewidget.h" -#include "coincontroldialog.h" +#include <qt/coincontroltreewidget.h> +#include <qt/coincontroldialog.h> CoinControlTreeWidget::CoinControlTreeWidget(QWidget *parent) : QTreeWidget(parent) diff --git a/src/qt/csvmodelwriter.cpp b/src/qt/csvmodelwriter.cpp index df59927782..672226ca11 100644 --- a/src/qt/csvmodelwriter.cpp +++ b/src/qt/csvmodelwriter.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "csvmodelwriter.h" +#include <qt/csvmodelwriter.h> #include <QAbstractItemModel> #include <QFile> diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index bb03c12f2d..6b7df2f2b9 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "editaddressdialog.h" -#include "ui_editaddressdialog.h" +#include <qt/editaddressdialog.h> +#include <qt/forms/ui_editaddressdialog.h> -#include "addresstablemodel.h" -#include "guiutil.h" +#include <qt/addresstablemodel.h> +#include <qt/guiutil.h> #include <QDataWidgetMapper> #include <QMessageBox> diff --git a/src/qt/forms/askpassphrasedialog.ui b/src/qt/forms/askpassphrasedialog.ui index a2105ecd0a..69803989cd 100644 --- a/src/qt/forms/askpassphrasedialog.ui +++ b/src/qt/forms/askpassphrasedialog.ui @@ -93,6 +93,13 @@ </widget> </item> <item row="3" column="1"> + <widget class="QCheckBox" name="toggleShowPasswordButton"> + <property name="text"> + <string>Show password</string> + </property> + </widget> + </item> + <item row="4" column="1"> <widget class="QLabel" name="capsLabel"> <property name="font"> <font> diff --git a/src/qt/forms/coincontroldialog.ui b/src/qt/forms/coincontroldialog.ui index 1ea00eb5c3..d1237ad283 100644 --- a/src/qt/forms/coincontroldialog.ui +++ b/src/qt/forms/coincontroldialog.ui @@ -487,7 +487,7 @@ <customwidget> <class>CoinControlTreeWidget</class> <extends>QTreeWidget</extends> - <header>coincontroltreewidget.h</header> + <header>qt/coincontroltreewidget.h</header> </customwidget> </customwidgets> <resources/> diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index 093e644bdc..bba822882e 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -1424,7 +1424,7 @@ <customwidget> <class>TrafficGraphWidget</class> <extends>QWidget</extends> - <header>trafficgraphwidget.h</header> + <header>qt/trafficgraphwidget.h</header> <container>1</container> <slots> <slot>clear()</slot> diff --git a/src/qt/forms/editaddressdialog.ui b/src/qt/forms/editaddressdialog.ui index c1aea36338..e968271107 100644 --- a/src/qt/forms/editaddressdialog.ui +++ b/src/qt/forms/editaddressdialog.ui @@ -71,7 +71,7 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> </customwidgets> <resources/> diff --git a/src/qt/forms/modaloverlay.ui b/src/qt/forms/modaloverlay.ui index 65a7a6c77e..fdc52dc455 100644 --- a/src/qt/forms/modaloverlay.ui +++ b/src/qt/forms/modaloverlay.ui @@ -367,7 +367,7 @@ QLabel { color: rgb(40,40,40); }</string> <customwidget> <class>ModalOverlay</class> <extends>QWidget</extends> - <header>modaloverlay.h</header> + <header>qt/modaloverlay.h</header> <container>1</container> </customwidget> </customwidgets> diff --git a/src/qt/forms/openuridialog.ui b/src/qt/forms/openuridialog.ui index 7fce858bd2..0e1048bc07 100644 --- a/src/qt/forms/openuridialog.ui +++ b/src/qt/forms/openuridialog.ui @@ -77,7 +77,7 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> </customwidgets> <resources/> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index e31bfee05e..a3721991ee 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -824,12 +824,12 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> <customwidget> <class>QValueComboBox</class> <extends>QComboBox</extends> - <header>qvaluecombobox.h</header> + <header>qt/qvaluecombobox.h</header> </customwidget> </customwidgets> <resources/> diff --git a/src/qt/forms/receivecoinsdialog.ui b/src/qt/forms/receivecoinsdialog.ui index 03fcb2fb50..58f030ebf0 100644 --- a/src/qt/forms/receivecoinsdialog.ui +++ b/src/qt/forms/receivecoinsdialog.ui @@ -28,23 +28,6 @@ <layout class="QVBoxLayout" name="verticalLayout_3"> <item> <layout class="QGridLayout" name="gridLayout"> - <item row="7" column="2"> - <widget class="QCheckBox" name="reuseAddress"> - <property name="toolTip"> - <string>Reuse one of the previously used receiving addresses. Reusing addresses has security and privacy issues. Do not use this unless re-generating a payment request made before.</string> - </property> - <property name="text"> - <string>R&euse an existing receiving address (not recommended)</string> - </property> - </widget> - </item> - <item row="7" column="0"> - <widget class="QLabel" name="label_4"> - <property name="text"> - <string/> - </property> - </widget> - </item> <item row="6" column="0"> <widget class="QLabel" name="label_3"> <property name="toolTip"> @@ -127,7 +110,7 @@ </property> </widget> </item> - <item row="8" column="2"> + <item row="7" column="2"> <layout class="QHBoxLayout" name="horizontalLayout"> <item> <widget class="QPushButton" name="receiveButton"> @@ -184,7 +167,7 @@ </item> </layout> </item> - <item row="8" column="0"> + <item row="7" column="0"> <widget class="QLabel" name="label_7"> <property name="text"> <string/> @@ -316,7 +299,7 @@ <customwidget> <class>BitcoinAmountField</class> <extends>QLineEdit</extends> - <header>bitcoinamountfield.h</header> + <header>qt/bitcoinamountfield.h</header> <container>1</container> </customwidget> </customwidgets> @@ -324,7 +307,6 @@ <tabstop>reqLabel</tabstop> <tabstop>reqAmount</tabstop> <tabstop>reqMessage</tabstop> - <tabstop>reuseAddress</tabstop> <tabstop>receiveButton</tabstop> <tabstop>clearButton</tabstop> <tabstop>recentRequestsView</tabstop> diff --git a/src/qt/forms/receiverequestdialog.ui b/src/qt/forms/receiverequestdialog.ui index 4163f4189c..dbe966b241 100644 --- a/src/qt/forms/receiverequestdialog.ui +++ b/src/qt/forms/receiverequestdialog.ui @@ -127,7 +127,7 @@ <customwidget> <class>QRImageWidget</class> <extends>QLabel</extends> - <header>receiverequestdialog.h</header> + <header>qt/receiverequestdialog.h</header> </customwidget> </customwidgets> <resources/> diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index a0e48334c1..195a5560f7 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -1108,10 +1108,10 @@ <item> <widget class="QCheckBox" name="optInRBF"> <property name="text"> - <string>Request Replace-By-Fee</string> + <string>Enable Replace-By-Fee</string> </property> <property name="toolTip"> - <string>Indicates that the sender may wish to replace this transaction with a new one paying higher fees (prior to being confirmed).</string> + <string>With Replace-By-Fee (BIP-125) you can increase a transaction's fee after it is sent. Without this, a higher fee may be recommended to compensate for increased transaction delay risk.</string> </property> </widget> </item> @@ -1264,12 +1264,12 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> <customwidget> <class>BitcoinAmountField</class> <extends>QLineEdit</extends> - <header>bitcoinamountfield.h</header> + <header>qt/bitcoinamountfield.h</header> <container>1</container> </customwidget> </customwidgets> diff --git a/src/qt/forms/sendcoinsentry.ui b/src/qt/forms/sendcoinsentry.ui index df06f36833..3c699abc6a 100644 --- a/src/qt/forms/sendcoinsentry.ui +++ b/src/qt/forms/sendcoinsentry.ui @@ -163,7 +163,7 @@ </widget> </item> <item row="2" column="1"> - <layout class="QHBoxLayout" name="horizontalLayoutAmount" stretch="0,1"> + <layout class="QHBoxLayout" name="horizontalLayoutAmount" stretch="0,1,0"> <item> <widget class="BitcoinAmountField" name="payAmount"/> </item> @@ -177,6 +177,13 @@ </property> </widget> </item> + <item> + <widget class="QPushButton" name="useAvailableBalanceButton"> + <property name="text"> + <string>Use available balance</string> + </property> + </widget> + </item> </layout> </item> <item row="3" column="0"> @@ -1248,12 +1255,12 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> <customwidget> <class>BitcoinAmountField</class> <extends>QLineEdit</extends> - <header>bitcoinamountfield.h</header> + <header>qt/bitcoinamountfield.h</header> </customwidget> </customwidgets> <tabstops> diff --git a/src/qt/forms/signverifymessagedialog.ui b/src/qt/forms/signverifymessagedialog.ui index 92f6430c51..c9ddd757c1 100644 --- a/src/qt/forms/signverifymessagedialog.ui +++ b/src/qt/forms/signverifymessagedialog.ui @@ -380,7 +380,7 @@ <customwidget> <class>QValidatedLineEdit</class> <extends>QLineEdit</extends> - <header>qvalidatedlineedit.h</header> + <header>qt/qvalidatedlineedit.h</header> </customwidget> </customwidgets> <resources> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index d520d7d4be..670d6108db 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -1,22 +1,21 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "guiutil.h" +#include <qt/guiutil.h> -#include "bitcoinaddressvalidator.h" -#include "bitcoinunits.h" -#include "qvalidatedlineedit.h" -#include "walletmodel.h" +#include <qt/bitcoinaddressvalidator.h> +#include <qt/bitcoinunits.h> +#include <qt/qvalidatedlineedit.h> +#include <qt/walletmodel.h> -#include "fs.h" -#include "primitives/transaction.h" -#include "init.h" -#include "policy/policy.h" -#include "protocol.h" -#include "script/script.h" -#include "script/standard.h" -#include "util.h" +#include <primitives/transaction.h> +#include <init.h> +#include <policy/policy.h> +#include <protocol.h> +#include <script/script.h> +#include <script/standard.h> +#include <util.h> #ifdef WIN32 #ifdef _WIN32_WINNT @@ -31,9 +30,9 @@ #ifndef NOMINMAX #define NOMINMAX #endif -#include "shellapi.h" -#include "shlobj.h" -#include "shlwapi.h" +#include <shellapi.h> +#include <shlobj.h> +#include <shlwapi.h> #endif #include <boost/scoped_array.hpp> @@ -984,6 +983,18 @@ QString formatNiceTimeOffset(qint64 secs) return timeBehindText; } +QString formatBytes(uint64_t bytes) +{ + if(bytes < 1024) + return QString(QObject::tr("%1 B")).arg(bytes); + if(bytes < 1024 * 1024) + return QString(QObject::tr("%1 KB")).arg(bytes / 1024); + if(bytes < 1024 * 1024 * 1024) + return QString(QObject::tr("%1 MB")).arg(bytes / 1024 / 1024); + + return QString(QObject::tr("%1 GB")).arg(bytes / 1024 / 1024 / 1024); +} + void ClickableLabel::mouseReleaseEvent(QMouseEvent *event) { Q_EMIT clicked(event->pos()); diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index d10818d0c8..ad0e22ccd6 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -1,12 +1,12 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_GUIUTIL_H #define BITCOIN_QT_GUIUTIL_H -#include "amount.h" -#include "fs.h" +#include <amount.h> +#include <fs.h> #include <QEvent> #include <QHeaderView> @@ -199,6 +199,8 @@ namespace GUIUtil QString formatNiceTimeOffset(qint64 secs); + QString formatBytes(uint64_t bytes); + class ClickableLabel : public QLabel { Q_OBJECT diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 0ff95d8502..21cda71ea4 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -1,18 +1,18 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "fs.h" -#include "intro.h" -#include "ui_intro.h" +#include <fs.h> +#include <qt/intro.h> +#include <qt/forms/ui_intro.h> -#include "guiutil.h" +#include <qt/guiutil.h> -#include "util.h" +#include <util.h> #include <QFileDialog> #include <QSettings> @@ -60,7 +60,7 @@ private: Intro *intro; }; -#include "intro.moc" +#include <qt/intro.moc> FreespaceChecker::FreespaceChecker(Intro *_intro) { @@ -214,7 +214,10 @@ bool Intro::pickDataDirectory() } dataDir = intro.getDataDirectory(); try { - TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir)); + if (TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir))) { + // If a new data directory has been created, make wallets subdirectory too + TryCreateDirectories(GUIUtil::qstringToBoostPath(dataDir) / "wallets"); + } break; } catch (const fs::filesystem_error&) { QMessageBox::critical(0, tr(PACKAGE_NAME), diff --git a/src/qt/macnotificationhandler.h b/src/qt/macnotificationhandler.h index 3a005c3c46..1e25d3e7a4 100644 --- a/src/qt/macnotificationhandler.h +++ b/src/qt/macnotificationhandler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index e32a0bdda8..b573dbe226 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "modaloverlay.h" -#include "ui_modaloverlay.h" +#include <qt/modaloverlay.h> +#include <qt/forms/ui_modaloverlay.h> -#include "guiutil.h" +#include <qt/guiutil.h> -#include "chainparams.h" +#include <chainparams.h> #include <QResizeEvent> #include <QPropertyAnimation> diff --git a/src/qt/modaloverlay.h b/src/qt/modaloverlay.h index cda23f9540..63465ab66e 100644 --- a/src/qt/modaloverlay.h +++ b/src/qt/modaloverlay.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/networkstyle.cpp b/src/qt/networkstyle.cpp index 4b81c54d36..2816633b0f 100644 --- a/src/qt/networkstyle.cpp +++ b/src/qt/networkstyle.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "networkstyle.h" +#include <qt/networkstyle.h> -#include "guiconstants.h" +#include <qt/guiconstants.h> #include <QApplication> diff --git a/src/qt/notificator.cpp b/src/qt/notificator.cpp index 937928315b..64a8e5d989 100644 --- a/src/qt/notificator.cpp +++ b/src/qt/notificator.cpp @@ -1,12 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "notificator.h" +#include <qt/notificator.h> #include <QApplication> #include <QByteArray> -#include <QIcon> #include <QImageWriter> #include <QMessageBox> #include <QMetaType> @@ -24,7 +23,7 @@ // #define __ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES 0 #ifdef Q_OS_MAC #include <ApplicationServices/ApplicationServices.h> -#include "macnotificationhandler.h" +#include <qt/macnotificationhandler.h> #endif diff --git a/src/qt/notificator.h b/src/qt/notificator.h index 67f2b1df69..3da24cd1a1 100644 --- a/src/qt/notificator.h +++ b/src/qt/notificator.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #define BITCOIN_QT_NOTIFICATOR_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <QIcon> diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index 3ee656d470..751cfa8b43 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "openuridialog.h" -#include "ui_openuridialog.h" +#include <qt/openuridialog.h> +#include <qt/forms/ui_openuridialog.h> -#include "guiutil.h" -#include "walletmodel.h" +#include <qt/guiutil.h> +#include <qt/walletmodel.h> #include <QUrl> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 53e2e5053c..5bef473c63 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "optionsdialog.h" -#include "ui_optionsdialog.h" +#include <qt/optionsdialog.h> +#include <qt/forms/ui_optionsdialog.h> -#include "bitcoinunits.h" -#include "guiutil.h" -#include "optionsmodel.h" +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> -#include "validation.h" // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS -#include "netbase.h" -#include "txdb.h" // for -dbcache defaults +#include <validation.h> // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS +#include <netbase.h> +#include <txdb.h> // for -dbcache defaults #include <QDataWidgetMapper> #include <QDir> @@ -338,7 +338,7 @@ QValidator::State ProxyAddressValidator::validate(QString &input, int &pos) cons { Q_UNUSED(pos); // Validate the proxy - CService serv(LookupNumeric(input.toStdString().c_str(), 9050)); + CService serv(LookupNumeric(input.toStdString().c_str(), DEFAULT_GUI_PROXY_PORT)); proxyType addrProxy = proxyType(serv, true); if (addrProxy.IsValid()) return QValidator::Acceptable; diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index f9f5823c05..faf9ff8959 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index fb8c60d100..4ade88d843 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -1,33 +1,34 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "optionsmodel.h" +#include <qt/optionsmodel.h> -#include "bitcoinunits.h" -#include "guiutil.h" +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> -#include "amount.h" -#include "init.h" -#include "validation.h" // For DEFAULT_SCRIPTCHECK_THREADS -#include "net.h" -#include "netbase.h" -#include "txdb.h" // for -dbcache defaults -#include "intro.h" +#include <init.h> +#include <validation.h> // For DEFAULT_SCRIPTCHECK_THREADS +#include <net.h> +#include <netbase.h> +#include <txdb.h> // for -dbcache defaults +#include <qt/intro.h> #ifdef ENABLE_WALLET -#include "wallet/wallet.h" -#include "wallet/walletdb.h" +#include <wallet/wallet.h> +#include <wallet/walletdb.h> #endif #include <QNetworkProxy> #include <QSettings> #include <QStringList> +const char *DEFAULT_GUI_PROXY_HOST = "127.0.0.1"; + OptionsModel::OptionsModel(QObject *parent, bool resetSettings) : QAbstractListModel(parent) { @@ -124,8 +125,8 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("fUseProxy")) settings.setValue("fUseProxy", false); - if (!settings.contains("addrProxy") || !settings.value("addrProxy").toString().contains(':')) - settings.setValue("addrProxy", "127.0.0.1:9050"); + if (!settings.contains("addrProxy")) + settings.setValue("addrProxy", QString("%1:%2").arg(DEFAULT_GUI_PROXY_HOST, DEFAULT_GUI_PROXY_PORT)); // Only try to set -proxy, if user has enabled fUseProxy if (settings.value("fUseProxy").toBool() && !gArgs.SoftSetArg("-proxy", settings.value("addrProxy").toString().toStdString())) addOverriddenOption("-proxy"); @@ -134,8 +135,8 @@ void OptionsModel::Init(bool resetSettings) if (!settings.contains("fUseSeparateProxyTor")) settings.setValue("fUseSeparateProxyTor", false); - if (!settings.contains("addrSeparateProxyTor") || !settings.value("addrSeparateProxyTor").toString().contains(':')) - settings.setValue("addrSeparateProxyTor", "127.0.0.1:9050"); + if (!settings.contains("addrSeparateProxyTor")) + settings.setValue("addrSeparateProxyTor", QString("%1:%2").arg(DEFAULT_GUI_PROXY_HOST, DEFAULT_GUI_PROXY_PORT)); // Only try to set -onion, if user has enabled fUseSeparateProxyTor if (settings.value("fUseSeparateProxyTor").toBool() && !gArgs.SoftSetArg("-onion", settings.value("addrSeparateProxyTor").toString().toStdString())) addOverriddenOption("-onion"); @@ -200,6 +201,33 @@ int OptionsModel::rowCount(const QModelIndex & parent) const return OptionIDRowCount; } +struct ProxySetting { + bool is_set; + QString ip; + QString port; +}; + +static ProxySetting GetProxySetting(QSettings &settings, const QString &name) +{ + static const ProxySetting default_val = {false, DEFAULT_GUI_PROXY_HOST, QString("%1").arg(DEFAULT_GUI_PROXY_PORT)}; + // Handle the case that the setting is not set at all + if (!settings.contains(name)) { + return default_val; + } + // contains IP at index 0 and port at index 1 + QStringList ip_port = settings.value(name).toString().split(":", QString::SkipEmptyParts); + if (ip_port.size() == 2) { + return {true, ip_port.at(0), ip_port.at(1)}; + } else { // Invalid: return default + return default_val; + } +} + +static void SetProxySetting(QSettings &settings, const QString &name, const ProxySetting &ip_port) +{ + settings.setValue(name, ip_port.ip + ":" + ip_port.port); +} + // read QSettings values and return them QVariant OptionsModel::data(const QModelIndex & index, int role) const { @@ -226,30 +254,18 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const // default proxy case ProxyUse: return settings.value("fUseProxy", false); - case ProxyIP: { - // contains IP at index 0 and port at index 1 - QStringList strlIpPort = settings.value("addrProxy").toString().split(":", QString::SkipEmptyParts); - return strlIpPort.at(0); - } - case ProxyPort: { - // contains IP at index 0 and port at index 1 - QStringList strlIpPort = settings.value("addrProxy").toString().split(":", QString::SkipEmptyParts); - return strlIpPort.at(1); - } + case ProxyIP: + return GetProxySetting(settings, "addrProxy").ip; + case ProxyPort: + return GetProxySetting(settings, "addrProxy").port; // separate Tor proxy case ProxyUseTor: return settings.value("fUseSeparateProxyTor", false); - case ProxyIPTor: { - // contains IP at index 0 and port at index 1 - QStringList strlIpPort = settings.value("addrSeparateProxyTor").toString().split(":", QString::SkipEmptyParts); - return strlIpPort.at(0); - } - case ProxyPortTor: { - // contains IP at index 0 and port at index 1 - QStringList strlIpPort = settings.value("addrSeparateProxyTor").toString().split(":", QString::SkipEmptyParts); - return strlIpPort.at(1); - } + case ProxyIPTor: + return GetProxySetting(settings, "addrSeparateProxyTor").ip; + case ProxyPortTor: + return GetProxySetting(settings, "addrSeparateProxyTor").port; #ifdef ENABLE_WALLET case SpendZeroConfChange: @@ -314,25 +330,19 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in } break; case ProxyIP: { - // contains current IP at index 0 and current port at index 1 - QStringList strlIpPort = settings.value("addrProxy").toString().split(":", QString::SkipEmptyParts); - // if that key doesn't exist or has a changed IP - if (!settings.contains("addrProxy") || strlIpPort.at(0) != value.toString()) { - // construct new value from new IP and current port - QString strNewValue = value.toString() + ":" + strlIpPort.at(1); - settings.setValue("addrProxy", strNewValue); + auto ip_port = GetProxySetting(settings, "addrProxy"); + if (!ip_port.is_set || ip_port.ip != value.toString()) { + ip_port.ip = value.toString(); + SetProxySetting(settings, "addrProxy", ip_port); setRestartRequired(true); } } break; case ProxyPort: { - // contains current IP at index 0 and current port at index 1 - QStringList strlIpPort = settings.value("addrProxy").toString().split(":", QString::SkipEmptyParts); - // if that key doesn't exist or has a changed port - if (!settings.contains("addrProxy") || strlIpPort.at(1) != value.toString()) { - // construct new value from current IP and new port - QString strNewValue = strlIpPort.at(0) + ":" + value.toString(); - settings.setValue("addrProxy", strNewValue); + auto ip_port = GetProxySetting(settings, "addrProxy"); + if (!ip_port.is_set || ip_port.port != value.toString()) { + ip_port.port = value.toString(); + SetProxySetting(settings, "addrProxy", ip_port); setRestartRequired(true); } } @@ -346,25 +356,19 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in } break; case ProxyIPTor: { - // contains current IP at index 0 and current port at index 1 - QStringList strlIpPort = settings.value("addrSeparateProxyTor").toString().split(":", QString::SkipEmptyParts); - // if that key doesn't exist or has a changed IP - if (!settings.contains("addrSeparateProxyTor") || strlIpPort.at(0) != value.toString()) { - // construct new value from new IP and current port - QString strNewValue = value.toString() + ":" + strlIpPort.at(1); - settings.setValue("addrSeparateProxyTor", strNewValue); + auto ip_port = GetProxySetting(settings, "addrSeparateProxyTor"); + if (!ip_port.is_set || ip_port.ip != value.toString()) { + ip_port.ip = value.toString(); + SetProxySetting(settings, "addrSeparateProxyTor", ip_port); setRestartRequired(true); } } break; case ProxyPortTor: { - // contains current IP at index 0 and current port at index 1 - QStringList strlIpPort = settings.value("addrSeparateProxyTor").toString().split(":", QString::SkipEmptyParts); - // if that key doesn't exist or has a changed port - if (!settings.contains("addrSeparateProxyTor") || strlIpPort.at(1) != value.toString()) { - // construct new value from current IP and new port - QString strNewValue = strlIpPort.at(0) + ":" + value.toString(); - settings.setValue("addrSeparateProxyTor", strNewValue); + auto ip_port = GetProxySetting(settings, "addrSeparateProxyTor"); + if (!ip_port.is_set || ip_port.port != value.toString()) { + ip_port.port = value.toString(); + SetProxySetting(settings, "addrSeparateProxyTor", ip_port); setRestartRequired(true); } } diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 0ac82a4148..3f50541eb4 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_OPTIONSMODEL_H #define BITCOIN_QT_OPTIONSMODEL_H -#include "amount.h" +#include <amount.h> #include <QAbstractListModel> @@ -13,6 +13,9 @@ QT_BEGIN_NAMESPACE class QNetworkProxy; QT_END_NAMESPACE +extern const char *DEFAULT_GUI_PROXY_HOST; +static constexpr unsigned short DEFAULT_GUI_PROXY_PORT = 9050; + /** Interface from Qt to configuration data structure for Bitcoin client. To Qt, the options are presented as a list with the different options laid out vertically. diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index ba1839e7b4..d1c9f17961 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -1,19 +1,19 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "overviewpage.h" -#include "ui_overviewpage.h" +#include <qt/overviewpage.h> +#include <qt/forms/ui_overviewpage.h> -#include "bitcoinunits.h" -#include "clientmodel.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "transactionfilterproxy.h" -#include "transactiontablemodel.h" -#include "walletmodel.h" +#include <qt/bitcoinunits.h> +#include <qt/clientmodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/transactionfilterproxy.h> +#include <qt/transactiontablemodel.h> +#include <qt/walletmodel.h> #include <QAbstractItemDelegate> #include <QPainter> @@ -106,7 +106,7 @@ public: const PlatformStyle *platformStyle; }; -#include "overviewpage.moc" +#include <qt/overviewpage.moc> OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) : QWidget(parent), diff --git a/src/qt/overviewpage.h b/src/qt/overviewpage.h index d76b651ce6..0ce9f98c8c 100644 --- a/src/qt/overviewpage.h +++ b/src/qt/overviewpage.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_OVERVIEWPAGE_H #define BITCOIN_QT_OVERVIEWPAGE_H -#include "amount.h" +#include <amount.h> #include <QWidget> #include <memory> diff --git a/src/qt/paymentrequestplus.cpp b/src/qt/paymentrequestplus.cpp index d3799f59ab..b0ef475b35 100644 --- a/src/qt/paymentrequestplus.cpp +++ b/src/qt/paymentrequestplus.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,9 +7,9 @@ // with some extra methods // -#include "paymentrequestplus.h" +#include <qt/paymentrequestplus.h> -#include "util.h" +#include <util.h> #include <stdexcept> @@ -194,8 +194,7 @@ bool PaymentRequestPlus::getMerchant(X509_STORE* certStore, QString& merchant) c qWarning() << "PaymentRequestPlus::getMerchant: SSL error: " << err.what(); } - if (website) - delete[] website; + delete[] website; X509_STORE_CTX_free(store_ctx); for (unsigned int i = 0; i < certs.size(); i++) X509_free(certs[i]); diff --git a/src/qt/paymentrequestplus.h b/src/qt/paymentrequestplus.h index a2fea3fdc6..be3923304f 100644 --- a/src/qt/paymentrequestplus.h +++ b/src/qt/paymentrequestplus.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,10 +7,10 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#include "paymentrequest.pb.h" +#include <qt/paymentrequest.pb.h> #pragma GCC diagnostic pop -#include "base58.h" +#include <base58.h> #include <openssl/x509.h> diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index 169684cf6d..dc729649b8 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -1,19 +1,19 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "paymentserver.h" +#include <qt/paymentserver.h> -#include "bitcoinunits.h" -#include "guiutil.h" -#include "optionsmodel.h" +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> -#include "base58.h" -#include "chainparams.h" -#include "policy/policy.h" -#include "ui_interface.h" -#include "util.h" -#include "wallet/wallet.h" +#include <base58.h> +#include <chainparams.h> +#include <policy/policy.h> +#include <ui_interface.h> +#include <util.h> +#include <wallet/wallet.h> #include <cstdlib> @@ -362,8 +362,7 @@ void PaymentServer::initNetManager() { if (!optionsModel) return; - if (netManager != nullptr) - delete netManager; + delete netManager; // netManager is used to fetch paymentrequests given in bitcoin: URIs netManager = new QNetworkAccessManager(this); @@ -637,27 +636,24 @@ void PaymentServer::fetchPaymentACK(CWallet* wallet, const SendCoinsRecipient& r // Create a new refund address, or re-use: QString account = tr("Refund from %1").arg(recipient.authenticatedMerchant); std::string strAccount = account.toStdString(); - std::set<CTxDestination> refundAddresses = wallet->GetAccountAddresses(strAccount); - if (!refundAddresses.empty()) { - CScript s = GetScriptForDestination(*refundAddresses.begin()); + CPubKey newKey; + if (wallet->GetKeyFromPool(newKey)) { + // BIP70 requests encode the scriptPubKey directly, so we are not restricted to address + // types supported by the receiver. As a result, we choose the address format we also + // use for change. Despite an actual payment and not change, this is a close match: + // it's the output type we use subject to privacy issues, but not restricted by what + // other software supports. + wallet->LearnRelatedScripts(newKey, g_change_type); + CTxDestination dest = GetDestinationForKey(newKey, g_change_type); + wallet->SetAddressBook(dest, strAccount, "refund"); + + CScript s = GetScriptForDestination(dest); payments::Output* refund_to = payment.add_refund_to(); refund_to->set_script(&s[0], s.size()); - } - else { - CPubKey newKey; - if (wallet->GetKeyFromPool(newKey)) { - CKeyID keyID = newKey.GetID(); - wallet->SetAddressBook(keyID, strAccount, "refund"); - - CScript s = GetScriptForDestination(keyID); - payments::Output* refund_to = payment.add_refund_to(); - refund_to->set_script(&s[0], s.size()); - } - else { - // This should never happen, because sending coins should have - // just unlocked the wallet and refilled the keypool. - qWarning() << "PaymentServer::fetchPaymentACK: Error getting refund key, refund_to not set"; - } + } else { + // This should never happen, because sending coins should have + // just unlocked the wallet and refilled the keypool. + qWarning() << "PaymentServer::fetchPaymentACK: Error getting refund key, refund_to not set"; } int length = payment.ByteSize(); diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 98b2364b92..e262fc6bf7 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -32,8 +32,8 @@ // sends them to the server. // -#include "paymentrequestplus.h" -#include "walletmodel.h" +#include <qt/paymentrequestplus.h> +#include <qt/walletmodel.h> #include <QObject> #include <QString> diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 42934f8055..0a57dcfca3 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "peertablemodel.h" +#include <qt/peertablemodel.h> -#include "clientmodel.h" -#include "guiconstants.h" -#include "guiutil.h" +#include <qt/clientmodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> -#include "validation.h" // for cs_main -#include "sync.h" +#include <validation.h> // for cs_main +#include <sync.h> #include <QDebug> #include <QList> @@ -33,6 +33,10 @@ bool NodeLessThan::operator()(const CNodeCombinedStats &left, const CNodeCombine return pLeft->cleanSubVer.compare(pRight->cleanSubVer) < 0; case PeerTableModel::Ping: return pLeft->dMinPing < pRight->dMinPing; + case PeerTableModel::Sent: + return pLeft->nSendBytes < pRight->nSendBytes; + case PeerTableModel::Received: + return pLeft->nRecvBytes < pRight->nRecvBytes; } return false; @@ -114,7 +118,7 @@ PeerTableModel::PeerTableModel(ClientModel *parent) : clientModel(parent), timer(0) { - columns << tr("NodeId") << tr("Node/Service") << tr("User Agent") << tr("Ping"); + columns << tr("NodeId") << tr("Node/Service") << tr("Ping") << tr("Sent") << tr("Received") << tr("User Agent"); priv.reset(new PeerTablePriv()); // default to unsorted priv->sortColumn = -1; @@ -173,10 +177,20 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const return QString::fromStdString(rec->nodeStats.cleanSubVer); case Ping: return GUIUtil::formatPingTime(rec->nodeStats.dMinPing); + case Sent: + return GUIUtil::formatBytes(rec->nodeStats.nSendBytes); + case Received: + return GUIUtil::formatBytes(rec->nodeStats.nRecvBytes); } } else if (role == Qt::TextAlignmentRole) { - if (index.column() == Ping) - return (QVariant)(Qt::AlignRight | Qt::AlignVCenter); + switch (index.column()) { + case Ping: + case Sent: + case Received: + return QVariant(Qt::AlignRight | Qt::AlignVCenter); + default: + return QVariant(); + } } return QVariant(); diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index cc47b67ec9..e3c9c6e5a3 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -1,12 +1,12 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_PEERTABLEMODEL_H #define BITCOIN_QT_PEERTABLEMODEL_H -#include "net_processing.h" // For CNodeStateStats -#include "net.h" +#include <net_processing.h> // For CNodeStateStats +#include <net.h> #include <QAbstractTableModel> #include <QStringList> @@ -55,8 +55,10 @@ public: enum ColumnIndex { NetNodeId = 0, Address = 1, - Subversion = 2, - Ping = 3 + Ping = 2, + Sent = 3, + Received = 4, + Subversion = 5 }; /** @name Methods overridden from QAbstractTableModel diff --git a/src/qt/platformstyle.cpp b/src/qt/platformstyle.cpp index 1f4e1a442f..fce71f661a 100644 --- a/src/qt/platformstyle.cpp +++ b/src/qt/platformstyle.cpp @@ -1,17 +1,15 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "platformstyle.h" +#include <qt/platformstyle.h> -#include "guiconstants.h" +#include <qt/guiconstants.h> #include <QApplication> #include <QColor> -#include <QIcon> #include <QImage> #include <QPalette> -#include <QPixmap> static const struct { const char *platformId; diff --git a/src/qt/qvalidatedlineedit.cpp b/src/qt/qvalidatedlineedit.cpp index 179ecdc8b3..8a13cd340f 100644 --- a/src/qt/qvalidatedlineedit.cpp +++ b/src/qt/qvalidatedlineedit.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "qvalidatedlineedit.h" +#include <qt/qvalidatedlineedit.h> -#include "bitcoinaddressvalidator.h" -#include "guiconstants.h" +#include <qt/bitcoinaddressvalidator.h> +#include <qt/guiconstants.h> QValidatedLineEdit::QValidatedLineEdit(QWidget *parent) : QLineEdit(parent), diff --git a/src/qt/qvaluecombobox.cpp b/src/qt/qvaluecombobox.cpp index a3b9b994ec..72c3a9d835 100644 --- a/src/qt/qvaluecombobox.cpp +++ b/src/qt/qvaluecombobox.cpp @@ -1,8 +1,8 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "qvaluecombobox.h" +#include <qt/qvaluecombobox.h> QValueComboBox::QValueComboBox(QWidget *parent) : QComboBox(parent), role(Qt::UserRole) diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 84f43266e1..e07f2b74ee 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -1,23 +1,21 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "receivecoinsdialog.h" -#include "ui_receivecoinsdialog.h" +#include <qt/receivecoinsdialog.h> +#include <qt/forms/ui_receivecoinsdialog.h> -#include "addressbookpage.h" -#include "addresstablemodel.h" -#include "bitcoinunits.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "receiverequestdialog.h" -#include "recentrequeststablemodel.h" -#include "walletmodel.h" +#include <qt/addressbookpage.h> +#include <qt/addresstablemodel.h> +#include <qt/bitcoinunits.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/receiverequestdialog.h> +#include <qt/recentrequeststablemodel.h> +#include <qt/walletmodel.h> #include <QAction> #include <QCursor> -#include <QItemSelection> #include <QMessageBox> #include <QScrollBar> #include <QTextDocument> @@ -106,7 +104,6 @@ void ReceiveCoinsDialog::clear() ui->reqAmount->clear(); ui->reqLabel->setText(""); ui->reqMessage->setText(""); - ui->reuseAddress->setChecked(false); updateDisplayUnit(); } @@ -135,25 +132,8 @@ void ReceiveCoinsDialog::on_receiveButton_clicked() QString address; QString label = ui->reqLabel->text(); - if(ui->reuseAddress->isChecked()) - { - /* Choose existing receiving address */ - AddressBookPage dlg(platformStyle, AddressBookPage::ForSelection, AddressBookPage::ReceivingTab, this); - dlg.setModel(model->getAddressTableModel()); - if(dlg.exec()) - { - address = dlg.getReturnValue(); - if(label.isEmpty()) /* If no label provided, use the previously used label */ - { - label = model->getAddressTableModel()->labelForAddress(address); - } - } else { - return; - } - } else { - /* Generate new receiving address */ - address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, ""); - } + /* Generate new receiving address */ + address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, ""); SendCoinsRecipient info(address, label, ui->reqAmount->value(), ui->reqMessage->text()); ReceiveRequestDialog *dialog = new ReceiveRequestDialog(this); diff --git a/src/qt/receivecoinsdialog.h b/src/qt/receivecoinsdialog.h index 385f98565c..931f5aa101 100644 --- a/src/qt/receivecoinsdialog.h +++ b/src/qt/receivecoinsdialog.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_RECEIVECOINSDIALOG_H #define BITCOIN_QT_RECEIVECOINSDIALOG_H -#include "guiutil.h" +#include <qt/guiutil.h> #include <QDialog> #include <QHeaderView> diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index 4aa6375d8a..209397ca0c 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -1,15 +1,14 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "receiverequestdialog.h" -#include "ui_receiverequestdialog.h" +#include <qt/receiverequestdialog.h> +#include <qt/forms/ui_receiverequestdialog.h> -#include "bitcoinunits.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "walletmodel.h" +#include <qt/bitcoinunits.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> #include <QClipboard> #include <QDrag> @@ -22,7 +21,7 @@ #endif #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" /* for USE_QRCODE */ +#include <config/bitcoin-config.h> /* for USE_QRCODE */ #endif #ifdef USE_QRCODE diff --git a/src/qt/receiverequestdialog.h b/src/qt/receiverequestdialog.h index 1a9b165237..21bbf1edb7 100644 --- a/src/qt/receiverequestdialog.h +++ b/src/qt/receiverequestdialog.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_RECEIVEREQUESTDIALOG_H #define BITCOIN_QT_RECEIVEREQUESTDIALOG_H -#include "walletmodel.h" +#include <qt/walletmodel.h> #include <QDialog> #include <QImage> diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 1c4f7aca86..0dd7d46960 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "recentrequeststablemodel.h" +#include <qt/recentrequeststablemodel.h> -#include "bitcoinunits.h" -#include "guiutil.h" -#include "optionsmodel.h" +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> -#include "clientversion.h" -#include "streams.h" +#include <clientversion.h> +#include <streams.h> RecentRequestsTableModel::RecentRequestsTableModel(CWallet *wallet, WalletModel *parent) : @@ -123,7 +123,7 @@ void RecentRequestsTableModel::updateAmountColumnTitle() /** Gets title for amount column including current display unit if optionsModel reference available. */ QString RecentRequestsTableModel::getAmountTitle() { - return (this->walletModel->getOptionsModel() != nullptr) ? tr("Requested") + " ("+BitcoinUnits::name(this->walletModel->getOptionsModel()->getDisplayUnit()) + ")" : ""; + return (this->walletModel->getOptionsModel() != nullptr) ? tr("Requested") + " ("+BitcoinUnits::shortName(this->walletModel->getOptionsModel()->getDisplayUnit()) + ")" : ""; } QModelIndex RecentRequestsTableModel::index(int row, int column, const QModelIndex &parent) const diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index 0c02968f92..ebad98cee8 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_RECENTREQUESTSTABLEMODEL_H #define BITCOIN_QT_RECENTREQUESTSTABLEMODEL_H -#include "walletmodel.h" +#include <qt/walletmodel.h> #include <QAbstractTableModel> #include <QStringList> diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index d895fc1663..c4b209a880 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1,23 +1,22 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "rpcconsole.h" -#include "ui_debugwindow.h" +#include <qt/rpcconsole.h> +#include <qt/forms/ui_debugwindow.h> -#include "bantablemodel.h" -#include "clientmodel.h" -#include "guiutil.h" -#include "platformstyle.h" -#include "chainparams.h" -#include "netbase.h" -#include "rpc/server.h" -#include "rpc/client.h" -#include "util.h" +#include <qt/bantablemodel.h> +#include <qt/clientmodel.h> +#include <qt/platformstyle.h> +#include <chainparams.h> +#include <netbase.h> +#include <rpc/server.h> +#include <rpc/client.h> +#include <util.h> #include <openssl/crypto.h> @@ -35,7 +34,6 @@ #include <QScrollBar> #include <QSettings> #include <QSignalMapper> -#include <QThread> #include <QTime> #include <QTimer> #include <QStringList> @@ -125,7 +123,7 @@ public: }; -#include "rpcconsole.moc" +#include <qt/rpcconsole.moc> /** * Split shell command line into a list of arguments and optionally execute the command(s). @@ -392,11 +390,37 @@ void RPCExecutor::request(const QString &command) { std::string result; std::string executableCommand = command.toStdString() + "\n"; + + // Catch the console-only-help command before RPC call is executed and reply with help text as-if a RPC reply. + if(executableCommand == "help-console\n") + { + Q_EMIT reply(RPCConsole::CMD_REPLY, QString(("\n" + "This console accepts RPC commands using the standard syntax.\n" + " example: getblockhash 0\n\n" + + "This console can also accept RPC commands using parenthesized syntax.\n" + " example: getblockhash(0)\n\n" + + "Commands may be nested when specified with the parenthesized syntax.\n" + " example: getblock(getblockhash(0) 1)\n\n" + + "A space or a comma can be used to delimit arguments for either syntax.\n" + " example: getblockhash 0\n" + " getblockhash,0\n\n" + + "Named results can be queried with a non-quoted key string in brackets.\n" + " example: getblock(getblockhash(0) true)[tx]\n\n" + + "Results without keys can be queried using an integer in brackets.\n" + " example: getblock(getblockhash(0),true)[tx][0]\n\n"))); + return; + } if(!RPCConsole::RPCExecuteCommandLine(result, executableCommand)) { Q_EMIT reply(RPCConsole::CMD_ERROR, QString("Parse error: unbalanced ' or \"")); return; } + Q_EMIT reply(RPCConsole::CMD_REPLY, QString::fromStdString(result)); } catch (UniValue& objError) @@ -645,6 +669,7 @@ void RPCConsole::setClientModel(ClientModel *model) wordList << ("help " + commandList[i]).c_str(); } + wordList << "help-console"; wordList.sort(); autoCompleter = new QCompleter(wordList, this); autoCompleter->setModelSorting(QCompleter::CaseSensitivelySortedModel); @@ -750,10 +775,11 @@ void RPCConsole::clear(bool clearHistory) message(CMD_REPLY, (tr("Welcome to the %1 RPC console.").arg(tr(PACKAGE_NAME)) + "<br>" + tr("Use up and down arrows to navigate history, and %1 to clear screen.").arg("<b>"+clsKey+"</b>") + "<br>" + - tr("Type <b>help</b> for an overview of available commands.")) + - "<br><span class=\"secwarning\">" + + tr("Type %1 for an overview of available commands.").arg("<b>help</b>") + "<br>" + + tr("For more information on using this console type %1.").arg("<b>help-console</b>") + + "<br><span class=\"secwarning\"><br>" + tr("WARNING: Scammers have been active, telling users to type commands here, stealing their wallet contents. Do not use this console without fully understanding the ramifications of a command.") + - "</span>", + "</span>"), true); } @@ -935,18 +961,6 @@ void RPCConsole::on_sldGraphRange_valueChanged(int value) setTrafficGraphRange(mins); } -QString RPCConsole::FormatBytes(quint64 bytes) -{ - if(bytes < 1024) - return QString(tr("%1 B")).arg(bytes); - if(bytes < 1024 * 1024) - return QString(tr("%1 KB")).arg(bytes / 1024); - if(bytes < 1024 * 1024 * 1024) - return QString(tr("%1 MB")).arg(bytes / 1024 / 1024); - - return QString(tr("%1 GB")).arg(bytes / 1024 / 1024 / 1024); -} - void RPCConsole::setTrafficGraphRange(int mins) { ui->trafficGraph->setGraphRangeMins(mins); @@ -955,8 +969,8 @@ void RPCConsole::setTrafficGraphRange(int mins) void RPCConsole::updateTrafficStats(quint64 totalBytesIn, quint64 totalBytesOut) { - ui->lblBytesIn->setText(FormatBytes(totalBytesIn)); - ui->lblBytesOut->setText(FormatBytes(totalBytesOut)); + ui->lblBytesIn->setText(GUIUtil::formatBytes(totalBytesIn)); + ui->lblBytesOut->setText(GUIUtil::formatBytes(totalBytesOut)); } void RPCConsole::peerSelected(const QItemSelection &selected, const QItemSelection &deselected) @@ -1050,8 +1064,8 @@ void RPCConsole::updateNodeDetail(const CNodeCombinedStats *stats) ui->peerServices->setText(GUIUtil::formatServicesStr(stats->nodeStats.nServices)); ui->peerLastSend->setText(stats->nodeStats.nLastSend ? GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nLastSend) : tr("never")); ui->peerLastRecv->setText(stats->nodeStats.nLastRecv ? GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nLastRecv) : tr("never")); - ui->peerBytesSent->setText(FormatBytes(stats->nodeStats.nSendBytes)); - ui->peerBytesRecv->setText(FormatBytes(stats->nodeStats.nRecvBytes)); + ui->peerBytesSent->setText(GUIUtil::formatBytes(stats->nodeStats.nSendBytes)); + ui->peerBytesRecv->setText(GUIUtil::formatBytes(stats->nodeStats.nRecvBytes)); ui->peerConnTime->setText(GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nTimeConnected)); ui->peerPingTime->setText(GUIUtil::formatPingTime(stats->nodeStats.dPingTime)); ui->peerPingWait->setText(GUIUtil::formatPingTime(stats->nodeStats.dPingWait)); diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index da06818f87..c41cbb0933 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -1,14 +1,14 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_RPCCONSOLE_H #define BITCOIN_QT_RPCCONSOLE_H -#include "guiutil.h" -#include "peertablemodel.h" +#include <qt/guiutil.h> +#include <qt/peertablemodel.h> -#include "net.h" +#include <net.h> #include <QWidget> #include <QCompleter> @@ -123,7 +123,6 @@ Q_SIGNALS: void cmdRequest(const QString &command); private: - static QString FormatBytes(quint64 bytes); void startExecutor(); void setTrafficGraphRange(int mins); /** show detailed information on ui about selected node */ diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 6309070fef..9fd61db70e 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -1,35 +1,32 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "sendcoinsdialog.h" -#include "ui_sendcoinsdialog.h" - -#include "addresstablemodel.h" -#include "bitcoinunits.h" -#include "clientmodel.h" -#include "coincontroldialog.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "sendcoinsentry.h" -#include "walletmodel.h" - -#include "base58.h" -#include "chainparams.h" -#include "wallet/coincontrol.h" -#include "validation.h" // mempool and minRelayTxFee -#include "ui_interface.h" -#include "txmempool.h" -#include "policy/fees.h" -#include "wallet/fees.h" +#include <qt/sendcoinsdialog.h> +#include <qt/forms/ui_sendcoinsdialog.h> + +#include <qt/addresstablemodel.h> +#include <qt/bitcoinunits.h> +#include <qt/clientmodel.h> +#include <qt/coincontroldialog.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/sendcoinsentry.h> + +#include <base58.h> +#include <chainparams.h> +#include <wallet/coincontrol.h> +#include <validation.h> // mempool and minRelayTxFee +#include <ui_interface.h> +#include <txmempool.h> +#include <policy/fees.h> +#include <wallet/fees.h> #include <QFontMetrics> -#include <QMessageBox> #include <QScrollBar> #include <QSettings> #include <QTextDocument> -#include <QTimer> static const std::array<int, 9> confTargets = { {2, 4, 6, 12, 24, 48, 144, 504, 1008} }; int getConfTargetForIndex(int index) { @@ -165,7 +162,7 @@ void SendCoinsDialog::setModel(WalletModel *_model) coinControlUpdateLabels(); // fee section - for (const int &n : confTargets) { + for (const int n : confTargets) { ui->confTargetSelector->addItem(tr("%1 (%2 blocks)").arg(GUIUtil::formatNiceTimeOffset(n*Params().GetConsensus().nPowTargetSpacing)).arg(n)); } connect(ui->confTargetSelector, SIGNAL(currentIndexChanged(int)), this, SLOT(updateSmartFeeLabel())); @@ -184,7 +181,7 @@ void SendCoinsDialog::setModel(WalletModel *_model) updateSmartFeeLabel(); // set default rbf checkbox state - ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); + ui->optInRBF->setCheckState(Qt::Checked); // set the smartfee-sliders default value (wallets default conf.target or last stored value) QSettings settings; @@ -339,15 +336,17 @@ void SendCoinsDialog::on_sendButton_clicked() } questionString.append(tr("Total Amount %1") .arg(BitcoinUnits::formatHtmlWithUnit(model->getOptionsModel()->getDisplayUnit(), totalAmount))); - questionString.append(QString("<span style='font-size:10pt;font-weight:normal;'><br />(=%2)</span>") + questionString.append(QString("<span style='font-size:10pt;font-weight:normal;'><br />(=%1)</span>") .arg(alternativeUnits.join(" " + tr("or") + "<br />"))); - if (ui->optInRBF->isChecked()) - { - questionString.append("<hr /><span>"); - questionString.append(tr("This transaction signals replaceability (optin-RBF).")); - questionString.append("</span>"); + questionString.append("<hr /><span>"); + if (ui->optInRBF->isChecked()) { + questionString.append(tr("You can increase the fee later (signals Replace-By-Fee, BIP-125).")); + } else { + questionString.append(tr("Not signalling Replace-By-Fee, BIP-125.")); } + questionString.append("</span>"); + SendConfirmationDialog confirmationDialog(tr("Confirm send coins"), questionString.arg(formatted.join("<br />")), SEND_CONFIRM_DELAY, this); @@ -402,6 +401,7 @@ SendCoinsEntry *SendCoinsDialog::addEntry() entry->setModel(model); ui->entries->addWidget(entry); connect(entry, SIGNAL(removeEntry(SendCoinsEntry*)), this, SLOT(removeEntry(SendCoinsEntry*))); + connect(entry, SIGNAL(useAvailableBalance(SendCoinsEntry*)), this, SLOT(useAvailableBalance(SendCoinsEntry*))); connect(entry, SIGNAL(payAmountChanged()), this, SLOT(coinControlUpdateLabels())); connect(entry, SIGNAL(subtractFeeFromAmountChanged()), this, SLOT(coinControlUpdateLabels())); @@ -599,6 +599,31 @@ void SendCoinsDialog::on_buttonMinimizeFee_clicked() minimizeFeeSection(true); } +void SendCoinsDialog::useAvailableBalance(SendCoinsEntry* entry) +{ + // Get CCoinControl instance if CoinControl is enabled or create a new one. + CCoinControl coin_control; + if (model->getOptionsModel()->getCoinControlFeatures()) { + coin_control = *CoinControlDialog::coinControl; + } + + // Calculate available amount to send. + CAmount amount = model->getBalance(&coin_control); + for (int i = 0; i < ui->entries->count(); ++i) { + SendCoinsEntry* e = qobject_cast<SendCoinsEntry*>(ui->entries->itemAt(i)->widget()); + if (e && !e->isHidden() && e != entry) { + amount -= e->getValue().amount; + } + } + + if (amount > 0) { + entry->checkSubtractFeeFromAmount(); + entry->setAmount(amount); + } else { + entry->setAmount(0); + } +} + void SendCoinsDialog::setMinimumFee() { ui->customFee->setValue(GetRequiredFee(1000)); diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 70b4aa5a03..7c27785d12 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_SENDCOINSDIALOG_H #define BITCOIN_QT_SENDCOINSDIALOG_H -#include "walletmodel.h" +#include <qt/walletmodel.h> #include <QDialog> #include <QMessageBox> @@ -76,6 +76,7 @@ private Q_SLOTS: void on_buttonChooseFee_clicked(); void on_buttonMinimizeFee_clicked(); void removeEntry(SendCoinsEntry* entry); + void useAvailableBalance(SendCoinsEntry* entry); void updateDisplayUnit(); void coinControlFeatureChanged(bool); void coinControlButtonClicked(); diff --git a/src/qt/sendcoinsentry.cpp b/src/qt/sendcoinsentry.cpp index bb0f47b21c..b7decbb69b 100644 --- a/src/qt/sendcoinsentry.cpp +++ b/src/qt/sendcoinsentry.cpp @@ -1,16 +1,15 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "sendcoinsentry.h" -#include "ui_sendcoinsentry.h" +#include <qt/sendcoinsentry.h> +#include <qt/forms/ui_sendcoinsentry.h> -#include "addressbookpage.h" -#include "addresstablemodel.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "walletmodel.h" +#include <qt/addressbookpage.h> +#include <qt/addresstablemodel.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> #include <QApplication> #include <QClipboard> @@ -48,6 +47,7 @@ SendCoinsEntry::SendCoinsEntry(const PlatformStyle *_platformStyle, QWidget *par connect(ui->deleteButton, SIGNAL(clicked()), this, SLOT(deleteClicked())); connect(ui->deleteButton_is, SIGNAL(clicked()), this, SLOT(deleteClicked())); connect(ui->deleteButton_s, SIGNAL(clicked()), this, SLOT(deleteClicked())); + connect(ui->useAvailableBalanceButton, SIGNAL(clicked()), this, SLOT(useAvailableBalanceClicked())); } SendCoinsEntry::~SendCoinsEntry() @@ -112,11 +112,21 @@ void SendCoinsEntry::clear() updateDisplayUnit(); } +void SendCoinsEntry::checkSubtractFeeFromAmount() +{ + ui->checkboxSubtractFeeFromAmount->setChecked(true); +} + void SendCoinsEntry::deleteClicked() { Q_EMIT removeEntry(this); } +void SendCoinsEntry::useAvailableBalanceClicked() +{ + Q_EMIT useAvailableBalance(this); +} + bool SendCoinsEntry::validate() { if (!model) @@ -228,6 +238,11 @@ void SendCoinsEntry::setAddress(const QString &address) ui->payAmount->setFocus(); } +void SendCoinsEntry::setAmount(const CAmount &amount) +{ + ui->payAmount->setValue(amount); +} + bool SendCoinsEntry::isClear() { return ui->payTo->text().isEmpty() && ui->payTo_is->text().isEmpty() && ui->payTo_s->text().isEmpty(); diff --git a/src/qt/sendcoinsentry.h b/src/qt/sendcoinsentry.h index a8be670c2a..a9fdd5938c 100644 --- a/src/qt/sendcoinsentry.h +++ b/src/qt/sendcoinsentry.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_SENDCOINSENTRY_H #define BITCOIN_QT_SENDCOINSENTRY_H -#include "walletmodel.h" +#include <qt/walletmodel.h> #include <QStackedWidget> @@ -38,6 +38,7 @@ public: void setValue(const SendCoinsRecipient &value); void setAddress(const QString &address); + void setAmount(const CAmount &amount); /** Set up the tab chain manually, as Qt messes up the tab chain by default in some cases * (issue https://bugreports.qt-project.org/browse/QTBUG-10907). @@ -48,14 +49,17 @@ public: public Q_SLOTS: void clear(); + void checkSubtractFeeFromAmount(); Q_SIGNALS: void removeEntry(SendCoinsEntry *entry); + void useAvailableBalance(SendCoinsEntry* entry); void payAmountChanged(); void subtractFeeFromAmountChanged(); private Q_SLOTS: void deleteClicked(); + void useAvailableBalanceClicked(); void on_payTo_textChanged(const QString &address); void on_addressBookButton_clicked(); void on_pasteButton_clicked(); diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index cba9d4da38..364dcd6f45 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -1,19 +1,19 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "signverifymessagedialog.h" -#include "ui_signverifymessagedialog.h" +#include <qt/signverifymessagedialog.h> +#include <qt/forms/ui_signverifymessagedialog.h> -#include "addressbookpage.h" -#include "guiutil.h" -#include "platformstyle.h" -#include "walletmodel.h" +#include <qt/addressbookpage.h> +#include <qt/guiutil.h> +#include <qt/platformstyle.h> +#include <qt/walletmodel.h> -#include "base58.h" -#include "init.h" -#include "validation.h" // For strMessageMagic -#include "wallet/wallet.h" +#include <base58.h> +#include <init.h> +#include <validation.h> // For strMessageMagic +#include <wallet/wallet.h> #include <string> #include <vector> diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index a1fbba963c..fa3bd18d5a 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -1,23 +1,23 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "splashscreen.h" +#include <qt/splashscreen.h> -#include "networkstyle.h" +#include <qt/networkstyle.h> -#include "clientversion.h" -#include "init.h" -#include "util.h" -#include "ui_interface.h" -#include "version.h" +#include <clientversion.h> +#include <init.h> +#include <util.h> +#include <ui_interface.h> +#include <version.h> #ifdef ENABLE_WALLET -#include "wallet/wallet.h" +#include <wallet/wallet.h> #endif #include <QApplication> diff --git a/src/qt/splashscreen.h b/src/qt/splashscreen.h index c6cfd503f7..e1568e406b 100644 --- a/src/qt/splashscreen.h +++ b/src/qt/splashscreen.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/compattests.cpp b/src/qt/test/compattests.cpp index 2a7284b5b2..3e601583c3 100644 --- a/src/qt/test/compattests.cpp +++ b/src/qt/test/compattests.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "paymentrequestplus.h" // this includes protobuf's port.h which defines its own bswap macos +#include <qt/paymentrequestplus.h> // this includes protobuf's port.h which defines its own bswap macos -#include "compattests.h" +#include <qt/test/compattests.h> -#include "compat/byteswap.h" +#include <compat/byteswap.h> void CompatTests::bswapTests() { diff --git a/src/qt/test/paymentservertests.cpp b/src/qt/test/paymentservertests.cpp index 273bd10487..6e80625123 100644 --- a/src/qt/test/paymentservertests.cpp +++ b/src/qt/test/paymentservertests.cpp @@ -1,18 +1,18 @@ -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "paymentservertests.h" +#include <qt/test/paymentservertests.h> -#include "optionsmodel.h" -#include "paymentrequestdata.h" +#include <qt/optionsmodel.h> +#include <qt/test/paymentrequestdata.h> -#include "amount.h" -#include "random.h" -#include "script/script.h" -#include "script/standard.h" -#include "util.h" -#include "utilstrencodings.h" +#include <amount.h> +#include <random.h> +#include <script/script.h> +#include <script/standard.h> +#include <util.h> +#include <utilstrencodings.h> #include <openssl/x509.h> #include <openssl/x509_vfy.h> diff --git a/src/qt/test/paymentservertests.h b/src/qt/test/paymentservertests.h index faf167f2c6..f022d2687b 100644 --- a/src/qt/test/paymentservertests.h +++ b/src/qt/test/paymentservertests.h @@ -1,11 +1,11 @@ -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_TEST_PAYMENTSERVERTESTS_H #define BITCOIN_QT_TEST_PAYMENTSERVERTESTS_H -#include "../paymentserver.h" +#include <qt/paymentserver.h> #include <QObject> #include <QTest> diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 70fdd4bf58..aaec15cc13 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -1,19 +1,19 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpcnestedtests.h" - -#include "chainparams.h" -#include "consensus/validation.h" -#include "fs.h" -#include "validation.h" -#include "rpc/register.h" -#include "rpc/server.h" -#include "rpcconsole.h" -#include "test/test_bitcoin.h" -#include "univalue.h" -#include "util.h" +#include <qt/test/rpcnestedtests.h> + +#include <chainparams.h> +#include <consensus/validation.h> +#include <fs.h> +#include <validation.h> +#include <rpc/register.h> +#include <rpc/server.h> +#include <qt/rpcconsole.h> +#include <test/test_bitcoin.h> +#include <univalue.h> +#include <util.h> #include <QDir> #include <QtGlobal> diff --git a/src/qt/test/rpcnestedtests.h b/src/qt/test/rpcnestedtests.h index 9ad409019f..0ce1c66f44 100644 --- a/src/qt/test/rpcnestedtests.h +++ b/src/qt/test/rpcnestedtests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <QObject> #include <QTest> -#include "txdb.h" -#include "txmempool.h" +#include <txdb.h> +#include <txmempool.h> class RPCNestedTests : public QObject { @@ -17,9 +17,6 @@ class RPCNestedTests : public QObject private Q_SLOTS: void rpcNestedTests(); - -private: - CCoinsViewDB *pcoinsdbview; }; #endif // BITCOIN_QT_TEST_RPC_NESTED_TESTS_H diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 4c04e67ccc..25ee66dc6b 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -1,20 +1,20 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "chainparams.h" -#include "rpcnestedtests.h" -#include "util.h" -#include "uritests.h" -#include "compattests.h" +#include <chainparams.h> +#include <qt/test/rpcnestedtests.h> +#include <util.h> +#include <qt/test/uritests.h> +#include <qt/test/compattests.h> #ifdef ENABLE_WALLET -#include "paymentservertests.h" -#include "wallettests.h" +#include <qt/test/paymentservertests.h> +#include <qt/test/wallettests.h> #endif #include <QApplication> diff --git a/src/qt/test/uritests.cpp b/src/qt/test/uritests.cpp index 8b53c0d5c7..8415250630 100644 --- a/src/qt/test/uritests.cpp +++ b/src/qt/test/uritests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2009-2014 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "uritests.h" +#include <qt/test/uritests.h> -#include "guiutil.h" -#include "walletmodel.h" +#include <qt/guiutil.h> +#include <qt/walletmodel.h> #include <QUrl> diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index eeae58bd05..a270e5de59 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -1,22 +1,22 @@ -#include "wallettests.h" +#include <qt/test/wallettests.h> -#include "qt/bitcoinamountfield.h" -#include "qt/callback.h" -#include "qt/optionsmodel.h" -#include "qt/platformstyle.h" -#include "qt/qvalidatedlineedit.h" -#include "qt/sendcoinsdialog.h" -#include "qt/sendcoinsentry.h" -#include "qt/transactiontablemodel.h" -#include "qt/transactionview.h" -#include "qt/walletmodel.h" -#include "test/test_bitcoin.h" -#include "validation.h" -#include "wallet/wallet.h" -#include "qt/overviewpage.h" -#include "qt/receivecoinsdialog.h" -#include "qt/recentrequeststablemodel.h" -#include "qt/receiverequestdialog.h" +#include <qt/bitcoinamountfield.h> +#include <qt/callback.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/qvalidatedlineedit.h> +#include <qt/sendcoinsdialog.h> +#include <qt/sendcoinsentry.h> +#include <qt/transactiontablemodel.h> +#include <qt/transactionview.h> +#include <qt/walletmodel.h> +#include <test/test_bitcoin.h> +#include <validation.h> +#include <wallet/wallet.h> +#include <qt/overviewpage.h> +#include <qt/receivecoinsdialog.h> +#include <qt/recentrequeststablemodel.h> +#include <qt/receiverequestdialog.h> #include <QAbstractButton> #include <QAction> @@ -149,6 +149,9 @@ void BumpFee(TransactionView& view, const uint256& txid, bool expectDisabled, st // src/qt/test/test_bitcoin-qt -platform cocoa # macOS void TestGUI() { + g_address_type = OUTPUT_TYPE_P2SH_SEGWIT; + g_change_type = OUTPUT_TYPE_P2SH_SEGWIT; + // Set up wallet and chain with 105 blocks (5 mature blocks for spending). TestChain100Setup test; for (int i = 0; i < 5; ++i) { @@ -161,10 +164,13 @@ void TestGUI() wallet.LoadWallet(firstRun); { LOCK(wallet.cs_wallet); - wallet.SetAddressBook(test.coinbaseKey.GetPubKey().GetID(), "", "receive"); + wallet.SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), g_address_type), "", "receive"); wallet.AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); } - wallet.ScanForWalletTransactions(chainActive.Genesis(), true); + { + LOCK(cs_main); + wallet.ScanForWalletTransactions(chainActive.Genesis(), nullptr, true); + } wallet.SetBroadcastTransactions(true); // Create widgets for sending coins and listing transactions. diff --git a/src/qt/trafficgraphwidget.cpp b/src/qt/trafficgraphwidget.cpp index 5bb863451f..f869799462 100644 --- a/src/qt/trafficgraphwidget.cpp +++ b/src/qt/trafficgraphwidget.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "trafficgraphwidget.h" -#include "clientmodel.h" +#include <qt/trafficgraphwidget.h> +#include <qt/clientmodel.h> #include <QPainter> #include <QColor> diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 74f5c774a0..c1d28be0ab 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -1,22 +1,22 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactiondesc.h" +#include <qt/transactiondesc.h> -#include "bitcoinunits.h" -#include "guiutil.h" -#include "paymentserver.h" -#include "transactionrecord.h" +#include <qt/bitcoinunits.h> +#include <qt/guiutil.h> +#include <qt/paymentserver.h> +#include <qt/transactionrecord.h> -#include "base58.h" -#include "consensus/consensus.h" -#include "validation.h" -#include "script/script.h" -#include "timedata.h" -#include "util.h" -#include "wallet/db.h" -#include "wallet/wallet.h" +#include <base58.h> +#include <consensus/consensus.h> +#include <validation.h> +#include <script/script.h> +#include <timedata.h> +#include <util.h> +#include <wallet/db.h> +#include <wallet/wallet.h> #include <stdint.h> #include <string> @@ -24,7 +24,7 @@ QString TransactionDesc::FormatTxStatus(const CWalletTx& wtx) { AssertLockHeld(cs_main); - if (!CheckFinalTx(wtx)) + if (!CheckFinalTx(*wtx.tx)) { if (wtx.tx->nLockTime < LOCKTIME_THRESHOLD) return tr("Open for %n more block(s)", "", wtx.tx->nLockTime - chainActive.Height()); diff --git a/src/qt/transactiondescdialog.cpp b/src/qt/transactiondescdialog.cpp index 65f163deb2..161fccd462 100644 --- a/src/qt/transactiondescdialog.cpp +++ b/src/qt/transactiondescdialog.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactiondescdialog.h" -#include "ui_transactiondescdialog.h" +#include <qt/transactiondescdialog.h> +#include <qt/forms/ui_transactiondescdialog.h> -#include "transactiontablemodel.h" +#include <qt/transactiontablemodel.h> #include <QModelIndex> diff --git a/src/qt/transactionfilterproxy.cpp b/src/qt/transactionfilterproxy.cpp index 44e4ef8238..39d03fa547 100644 --- a/src/qt/transactionfilterproxy.cpp +++ b/src/qt/transactionfilterproxy.cpp @@ -1,16 +1,14 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactionfilterproxy.h" +#include <qt/transactionfilterproxy.h> -#include "transactiontablemodel.h" -#include "transactionrecord.h" +#include <qt/transactiontablemodel.h> +#include <qt/transactionrecord.h> #include <cstdlib> -#include <QDateTime> - // Earliest date that can be represented (far in the past) const QDateTime TransactionFilterProxy::MIN_DATE = QDateTime::fromTime_t(0); // Last date that can be represented (far in the future) @@ -20,7 +18,7 @@ TransactionFilterProxy::TransactionFilterProxy(QObject *parent) : QSortFilterProxyModel(parent), dateFrom(MIN_DATE), dateTo(MAX_DATE), - addrPrefix(), + m_search_string(), typeFilter(ALL_TYPES), watchOnlyFilter(WatchOnlyFilter_All), minAmount(0), @@ -38,6 +36,7 @@ bool TransactionFilterProxy::filterAcceptsRow(int sourceRow, const QModelIndex & bool involvesWatchAddress = index.data(TransactionTableModel::WatchonlyRole).toBool(); QString address = index.data(TransactionTableModel::AddressRole).toString(); QString label = index.data(TransactionTableModel::LabelRole).toString(); + QString txid = index.data(TransactionTableModel::TxIDRole).toString(); qint64 amount = llabs(index.data(TransactionTableModel::AmountRole).toLongLong()); int status = index.data(TransactionTableModel::StatusRole).toInt(); @@ -51,8 +50,11 @@ bool TransactionFilterProxy::filterAcceptsRow(int sourceRow, const QModelIndex & return false; if(datetime < dateFrom || datetime > dateTo) return false; - if (!address.contains(addrPrefix, Qt::CaseInsensitive) && !label.contains(addrPrefix, Qt::CaseInsensitive)) + if (!address.contains(m_search_string, Qt::CaseInsensitive) && + ! label.contains(m_search_string, Qt::CaseInsensitive) && + ! txid.contains(m_search_string, Qt::CaseInsensitive)) { return false; + } if(amount < minAmount) return false; @@ -66,9 +68,10 @@ void TransactionFilterProxy::setDateRange(const QDateTime &from, const QDateTime invalidateFilter(); } -void TransactionFilterProxy::setAddressPrefix(const QString &_addrPrefix) +void TransactionFilterProxy::setSearchString(const QString &search_string) { - this->addrPrefix = _addrPrefix; + if (m_search_string == search_string) return; + m_search_string = search_string; invalidateFilter(); } diff --git a/src/qt/transactionfilterproxy.h b/src/qt/transactionfilterproxy.h index 7db02cd61f..226cf3f3b5 100644 --- a/src/qt/transactionfilterproxy.h +++ b/src/qt/transactionfilterproxy.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_TRANSACTIONFILTERPROXY_H #define BITCOIN_QT_TRANSACTIONFILTERPROXY_H -#include "amount.h" +#include <amount.h> #include <QDateTime> #include <QSortFilterProxyModel> @@ -35,7 +35,7 @@ public: }; void setDateRange(const QDateTime &from, const QDateTime &to); - void setAddressPrefix(const QString &addrPrefix); + void setSearchString(const QString &); /** @note Type filter takes a bit field created with TYPE() or ALL_TYPES */ @@ -57,7 +57,7 @@ protected: private: QDateTime dateFrom; QDateTime dateTo; - QString addrPrefix; + QString m_search_string; quint32 typeFilter; WatchOnlyFilter watchOnlyFilter; CAmount minAmount; diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index d40ffd22cd..de3e885e8f 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactionrecord.h" +#include <qt/transactionrecord.h> -#include "base58.h" -#include "consensus/consensus.h" -#include "validation.h" -#include "timedata.h" -#include "wallet/wallet.h" +#include <base58.h> +#include <consensus/consensus.h> +#include <validation.h> +#include <timedata.h> +#include <wallet/wallet.h> #include <stdint.h> @@ -182,7 +182,7 @@ void TransactionRecord::updateStatus(const CWalletTx &wtx) status.depth = wtx.GetDepthInMainChain(); status.cur_num_blocks = chainActive.Height(); - if (!CheckFinalTx(wtx)) + if (!CheckFinalTx(*wtx.tx)) { if (wtx.tx->nLockTime < LOCKTIME_THRESHOLD) { diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index a26e676142..29a3cd8de7 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -1,12 +1,12 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_TRANSACTIONRECORD_H #define BITCOIN_QT_TRANSACTIONRECORD_H -#include "amount.h" -#include "uint256.h" +#include <amount.h> +#include <uint256.h> #include <QList> #include <QString> diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 59cef555b1..626d4c0bdc 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -1,24 +1,24 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactiontablemodel.h" +#include <qt/transactiontablemodel.h> -#include "addresstablemodel.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "transactiondesc.h" -#include "transactionrecord.h" -#include "walletmodel.h" +#include <qt/addresstablemodel.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/transactiondesc.h> +#include <qt/transactionrecord.h> +#include <qt/walletmodel.h> -#include "core_io.h" -#include "validation.h" -#include "sync.h" -#include "uint256.h" -#include "util.h" -#include "wallet/wallet.h" +#include <core_io.h> +#include <validation.h> +#include <sync.h> +#include <uint256.h> +#include <util.h> +#include <wallet/wallet.h> #include <QColor> #include <QDateTime> @@ -80,10 +80,10 @@ public: cachedWallet.clear(); { LOCK2(cs_main, wallet->cs_wallet); - for(std::map<uint256, CWalletTx>::iterator it = wallet->mapWallet.begin(); it != wallet->mapWallet.end(); ++it) + for (const auto& entry : wallet->mapWallet) { - if(TransactionRecord::showTransaction(it->second)) - cachedWallet.append(TransactionRecord::decomposeTransaction(wallet, it->second)); + if (TransactionRecord::showTransaction(entry.second)) + cachedWallet.append(TransactionRecord::decomposeTransaction(wallet, entry.second)); } } } @@ -230,7 +230,7 @@ public: std::map<uint256, CWalletTx>::iterator mi = wallet->mapWallet.find(rec->hash); if(mi != wallet->mapWallet.end()) { - std::string strHex = EncodeHexTx(static_cast<CTransaction>(mi->second)); + std::string strHex = EncodeHexTx(*mi->second.tx); return QString::fromStdString(strHex); } return QString(); diff --git a/src/qt/transactiontablemodel.h b/src/qt/transactiontablemodel.h index b1f81498b2..8f58962d17 100644 --- a/src/qt/transactiontablemodel.h +++ b/src/qt/transactiontablemodel.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_TRANSACTIONTABLEMODEL_H #define BITCOIN_QT_TRANSACTIONTABLEMODEL_H -#include "bitcoinunits.h" +#include <qt/bitcoinunits.h> #include <QAbstractTableModel> #include <QStringList> diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 39dfdb587c..fa43ab750a 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -1,24 +1,23 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "transactionview.h" +#include <qt/transactionview.h> -#include "addresstablemodel.h" -#include "bitcoinunits.h" -#include "csvmodelwriter.h" -#include "editaddressdialog.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "platformstyle.h" -#include "sendcoinsdialog.h" -#include "transactiondescdialog.h" -#include "transactionfilterproxy.h" -#include "transactionrecord.h" -#include "transactiontablemodel.h" -#include "walletmodel.h" +#include <qt/addresstablemodel.h> +#include <qt/bitcoinunits.h> +#include <qt/csvmodelwriter.h> +#include <qt/editaddressdialog.h> +#include <qt/optionsmodel.h> +#include <qt/platformstyle.h> +#include <qt/sendcoinsdialog.h> +#include <qt/transactiondescdialog.h> +#include <qt/transactionfilterproxy.h> +#include <qt/transactionrecord.h> +#include <qt/transactiontablemodel.h> +#include <qt/walletmodel.h> -#include "ui_interface.h" +#include <ui_interface.h> #include <QComboBox> #include <QDateTimeEdit> @@ -95,11 +94,11 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa hlayout->addWidget(typeWidget); - addressWidget = new QLineEdit(this); + search_widget = new QLineEdit(this); #if QT_VERSION >= 0x040700 - addressWidget->setPlaceholderText(tr("Enter address or label to search")); + search_widget->setPlaceholderText(tr("Enter address, transaction id, or label to search")); #endif - hlayout->addWidget(addressWidget); + hlayout->addWidget(search_widget); amountWidget = new QLineEdit(this); #if QT_VERSION >= 0x040700 @@ -187,8 +186,8 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa connect(watchOnlyWidget, SIGNAL(activated(int)), this, SLOT(chooseWatchonly(int))); connect(amountWidget, SIGNAL(textChanged(QString)), amount_typing_delay, SLOT(start())); connect(amount_typing_delay, SIGNAL(timeout()), this, SLOT(changedAmount())); - connect(addressWidget, SIGNAL(textChanged(QString)), prefix_typing_delay, SLOT(start())); - connect(prefix_typing_delay, SIGNAL(timeout()), this, SLOT(changedPrefix())); + connect(search_widget, SIGNAL(textChanged(QString)), prefix_typing_delay, SLOT(start())); + connect(prefix_typing_delay, SIGNAL(timeout()), this, SLOT(changedSearch())); connect(view, SIGNAL(doubleClicked(QModelIndex)), this, SIGNAL(doubleClicked(QModelIndex))); connect(view, SIGNAL(customContextMenuRequested(QPoint)), this, SLOT(contextualMenu(QPoint))); @@ -326,11 +325,11 @@ void TransactionView::chooseWatchonly(int idx) (TransactionFilterProxy::WatchOnlyFilter)watchOnlyWidget->itemData(idx).toInt()); } -void TransactionView::changedPrefix() +void TransactionView::changedSearch() { if(!transactionProxyModel) return; - transactionProxyModel->setAddressPrefix(addressWidget->text()); + transactionProxyModel->setSearchString(search_widget->text()); } void TransactionView::changedAmount() @@ -366,7 +365,7 @@ void TransactionView::exportClicked() // name, column, role writer.setModel(transactionProxyModel); writer.addColumn(tr("Confirmed"), 0, TransactionTableModel::ConfirmedRole); - if (model && model->haveWatchOnly()) + if (model->haveWatchOnly()) writer.addColumn(tr("Watch-only"), TransactionTableModel::Watchonly); writer.addColumn(tr("Date"), 0, TransactionTableModel::DateRole); writer.addColumn(tr("Type"), TransactionTableModel::Type, Qt::EditRole); diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index 5b4cfd4a88..82e929b53f 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_TRANSACTIONVIEW_H #define BITCOIN_QT_TRANSACTIONVIEW_H -#include "guiutil.h" +#include <qt/guiutil.h> #include <QWidget> #include <QKeyEvent> @@ -66,7 +66,7 @@ private: QComboBox *dateWidget; QComboBox *typeWidget; QComboBox *watchOnlyWidget; - QLineEdit *addressWidget; + QLineEdit *search_widget; QLineEdit *amountWidget; QMenu *contextMenu; @@ -113,7 +113,7 @@ public Q_SLOTS: void chooseType(int idx); void chooseWatchonly(int idx); void changedAmount(); - void changedPrefix(); + void changedSearch(); void exportClicked(); void focusTransaction(const QModelIndex&); diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 5d8c23d13c..5d4a92f760 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -1,25 +1,25 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "utilitydialog.h" +#include <qt/utilitydialog.h> -#include "ui_helpmessagedialog.h" +#include <qt/forms/ui_helpmessagedialog.h> -#include "bitcoingui.h" -#include "clientmodel.h" -#include "guiconstants.h" -#include "intro.h" -#include "paymentrequestplus.h" -#include "guiutil.h" +#include <qt/bitcoingui.h> +#include <qt/clientmodel.h> +#include <qt/guiconstants.h> +#include <qt/intro.h> +#include <qt/paymentrequestplus.h> +#include <qt/guiutil.h> -#include "clientversion.h" -#include "init.h" -#include "util.h" +#include <clientversion.h> +#include <init.h> +#include <util.h> #include <stdio.h> diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index 738eeed136..d43d9a82c0 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 714a594318..c0b9d04269 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "walletframe.h" +#include <qt/walletframe.h> -#include "bitcoingui.h" -#include "walletview.h" +#include <qt/bitcoingui.h> +#include <qt/walletview.h> #include <cassert> #include <cstdio> diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 53b1c2967c..494b46905d 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -1,33 +1,33 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "walletmodel.h" - -#include "addresstablemodel.h" -#include "consensus/validation.h" -#include "guiconstants.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "paymentserver.h" -#include "recentrequeststablemodel.h" -#include "sendcoinsdialog.h" -#include "transactiontablemodel.h" - -#include "base58.h" -#include "chain.h" -#include "keystore.h" -#include "validation.h" -#include "net.h" // for g_connman -#include "policy/fees.h" -#include "policy/rbf.h" -#include "sync.h" -#include "ui_interface.h" -#include "util.h" // for GetBoolArg -#include "wallet/coincontrol.h" -#include "wallet/feebumper.h" -#include "wallet/wallet.h" -#include "wallet/walletdb.h" // for BackupWallet +#include <qt/walletmodel.h> + +#include <qt/addresstablemodel.h> +#include <consensus/validation.h> +#include <qt/guiconstants.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/paymentserver.h> +#include <qt/recentrequeststablemodel.h> +#include <qt/sendcoinsdialog.h> +#include <qt/transactiontablemodel.h> + +#include <base58.h> +#include <chain.h> +#include <keystore.h> +#include <validation.h> +#include <net.h> // for g_connman +#include <policy/fees.h> +#include <policy/rbf.h> +#include <sync.h> +#include <ui_interface.h> +#include <util.h> // for GetBoolArg +#include <wallet/coincontrol.h> +#include <wallet/feebumper.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> // for BackupWallet #include <stdint.h> @@ -659,45 +659,39 @@ bool WalletModel::abandonTransaction(uint256 hash) const bool WalletModel::transactionCanBeBumped(uint256 hash) const { - LOCK2(cs_main, wallet->cs_wallet); - const CWalletTx *wtx = wallet->GetWalletTx(hash); - return wtx && SignalsOptInRBF(*wtx) && !wtx->mapValue.count("replaced_by_txid"); + return feebumper::TransactionCanBeBumped(wallet, hash); } bool WalletModel::bumpFee(uint256 hash) { - std::unique_ptr<CFeeBumper> feeBump; - { - CCoinControl coin_control; - coin_control.signalRbf = true; - LOCK2(cs_main, wallet->cs_wallet); - feeBump.reset(new CFeeBumper(wallet, hash, coin_control, 0)); - } - if (feeBump->getResult() != BumpFeeResult::OK) - { + CCoinControl coin_control; + coin_control.signalRbf = true; + std::vector<std::string> errors; + CAmount old_fee; + CAmount new_fee; + CMutableTransaction mtx; + if (feebumper::CreateTransaction(wallet, hash, coin_control, 0 /* totalFee */, errors, old_fee, new_fee, mtx) != feebumper::Result::OK) { QMessageBox::critical(0, tr("Fee bump error"), tr("Increasing transaction fee failed") + "<br />(" + - (feeBump->getErrors().size() ? QString::fromStdString(feeBump->getErrors()[0]) : "") +")"); + (errors.size() ? QString::fromStdString(errors[0]) : "") +")"); return false; } // allow a user based fee verification QString questionString = tr("Do you want to increase the fee?"); questionString.append("<br />"); - CAmount oldFee = feeBump->getOldFee(); - CAmount newFee = feeBump->getNewFee(); questionString.append("<table style=\"text-align: left;\">"); questionString.append("<tr><td>"); questionString.append(tr("Current fee:")); questionString.append("</td><td>"); - questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), oldFee)); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), old_fee)); questionString.append("</td></tr><tr><td>"); questionString.append(tr("Increase:")); questionString.append("</td><td>"); - questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee - oldFee)); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), new_fee - old_fee)); questionString.append("</td></tr><tr><td>"); questionString.append(tr("New fee:")); questionString.append("</td><td>"); - questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee)); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), new_fee)); questionString.append("</td></tr></table>"); SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); confirmationDialog.exec(); @@ -715,23 +709,15 @@ bool WalletModel::bumpFee(uint256 hash) } // sign bumped transaction - bool res = false; - { - LOCK2(cs_main, wallet->cs_wallet); - res = feeBump->signTransaction(wallet); - } - if (!res) { + if (!feebumper::SignTransaction(wallet, mtx)) { QMessageBox::critical(0, tr("Fee bump error"), tr("Can't sign transaction.")); return false; } // commit the bumped transaction - { - LOCK2(cs_main, wallet->cs_wallet); - res = feeBump->commit(wallet); - } - if(!res) { + uint256 txid; + if (feebumper::CommitTransaction(wallet, hash, std::move(mtx), errors, txid) != feebumper::Result::OK) { QMessageBox::critical(0, tr("Fee bump error"), tr("Could not commit transaction") + "<br />(" + - QString::fromStdString(feeBump->getErrors()[0])+")"); + QString::fromStdString(errors[0])+")"); return false; } return true; @@ -751,8 +737,3 @@ int WalletModel::getDefaultConfirmTarget() const { return nTxConfirmTarget; } - -bool WalletModel::getDefaultWalletRbf() const -{ - return fWalletRbf; -} diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 05733f8272..7d7e4f160e 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -1,14 +1,14 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_WALLETMODEL_H #define BITCOIN_QT_WALLETMODEL_H -#include "paymentrequestplus.h" -#include "walletmodeltransaction.h" +#include <qt/paymentrequestplus.h> +#include <qt/walletmodeltransaction.h> -#include "support/allocators/secure.h" +#include <support/allocators/secure.h> #include <map> #include <vector> @@ -216,8 +216,6 @@ public: int getDefaultConfirmTarget() const; - bool getDefaultWalletRbf() const; - private: CWallet *wallet; bool fHaveWatchOnly; diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index eae2c27f8a..4b2bef2690 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -1,16 +1,15 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "walletmodeltransaction.h" +#include <qt/walletmodeltransaction.h> -#include "policy/policy.h" -#include "wallet/wallet.h" +#include <policy/policy.h> +#include <wallet/wallet.h> WalletModelTransaction::WalletModelTransaction(const QList<SendCoinsRecipient> &_recipients) : recipients(_recipients), walletTransaction(0), - keyChange(0), fee(0) { walletTransaction = new CWalletTx(); @@ -18,7 +17,6 @@ WalletModelTransaction::WalletModelTransaction(const QList<SendCoinsRecipient> & WalletModelTransaction::~WalletModelTransaction() { - delete keyChange; delete walletTransaction; } @@ -34,7 +32,7 @@ CWalletTx *WalletModelTransaction::getTransaction() const unsigned int WalletModelTransaction::getTransactionSize() { - return (!walletTransaction ? 0 : ::GetVirtualTransactionSize(*walletTransaction)); + return (!walletTransaction ? 0 : ::GetVirtualTransactionSize(*walletTransaction->tx)); } CAmount WalletModelTransaction::getTransactionFee() const @@ -91,10 +89,10 @@ CAmount WalletModelTransaction::getTotalTransactionAmount() const void WalletModelTransaction::newPossibleKeyChange(CWallet *wallet) { - keyChange = new CReserveKey(wallet); + keyChange.reset(new CReserveKey(wallet)); } CReserveKey *WalletModelTransaction::getPossibleKeyChange() { - return keyChange; + return keyChange.get(); } diff --git a/src/qt/walletmodeltransaction.h b/src/qt/walletmodeltransaction.h index d7ecd7aa8c..cd531dba4b 100644 --- a/src/qt/walletmodeltransaction.h +++ b/src/qt/walletmodeltransaction.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2014 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_WALLETMODELTRANSACTION_H #define BITCOIN_QT_WALLETMODELTRANSACTION_H -#include "walletmodel.h" +#include <qt/walletmodel.h> #include <QObject> @@ -40,7 +40,7 @@ public: private: QList<SendCoinsRecipient> recipients; CWalletTx *walletTransaction; - CReserveKey *keyChange; + std::unique_ptr<CReserveKey> keyChange; CAmount fee; }; diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 971f5e0e1a..7eced9289d 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -1,25 +1,25 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "walletview.h" - -#include "addressbookpage.h" -#include "askpassphrasedialog.h" -#include "bitcoingui.h" -#include "clientmodel.h" -#include "guiutil.h" -#include "optionsmodel.h" -#include "overviewpage.h" -#include "platformstyle.h" -#include "receivecoinsdialog.h" -#include "sendcoinsdialog.h" -#include "signverifymessagedialog.h" -#include "transactiontablemodel.h" -#include "transactionview.h" -#include "walletmodel.h" - -#include "ui_interface.h" +#include <qt/walletview.h> + +#include <qt/addressbookpage.h> +#include <qt/askpassphrasedialog.h> +#include <qt/bitcoingui.h> +#include <qt/clientmodel.h> +#include <qt/guiutil.h> +#include <qt/optionsmodel.h> +#include <qt/overviewpage.h> +#include <qt/platformstyle.h> +#include <qt/receivecoinsdialog.h> +#include <qt/sendcoinsdialog.h> +#include <qt/signverifymessagedialog.h> +#include <qt/transactiontablemodel.h> +#include <qt/transactionview.h> +#include <qt/walletmodel.h> + +#include <ui_interface.h> #include <QAction> #include <QActionGroup> @@ -122,8 +122,8 @@ void WalletView::setWalletModel(WalletModel *_walletModel) overviewPage->setWalletModel(_walletModel); receiveCoinsPage->setModel(_walletModel); sendCoinsPage->setModel(_walletModel); - usedReceivingAddressesPage->setModel(_walletModel->getAddressTableModel()); - usedSendingAddressesPage->setModel(_walletModel->getAddressTableModel()); + usedReceivingAddressesPage->setModel(_walletModel ? _walletModel->getAddressTableModel() : nullptr); + usedSendingAddressesPage->setModel(_walletModel ? _walletModel->getAddressTableModel() : nullptr); if (_walletModel) { diff --git a/src/qt/walletview.h b/src/qt/walletview.h index c1f8422f0c..30d68e4eff 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -1,11 +1,11 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_QT_WALLETVIEW_H #define BITCOIN_QT_WALLETVIEW_H -#include "amount.h" +#include <amount.h> #include <QStackedWidget> diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index d78d9a2358..1e7a76efc0 100644 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "winshutdownmonitor.h" +#include <qt/winshutdownmonitor.h> #if defined(Q_OS_WIN) && QT_VERSION >= 0x050000 -#include "init.h" -#include "util.h" +#include <init.h> +#include <util.h> #include <windows.h> diff --git a/src/random.cpp b/src/random.cpp index 7e0e94439e..a845526d8a 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -1,18 +1,18 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "random.h" +#include <random.h> -#include "crypto/sha512.h" -#include "support/cleanse.h" +#include <crypto/sha512.h> +#include <support/cleanse.h> #ifdef WIN32 -#include "compat.h" // for Windows API +#include <compat.h> // for Windows API #include <wincrypt.h> #endif -#include "util.h" // for LogPrint() -#include "utilstrencodings.h" // for GetTime() +#include <util.h> // for LogPrint() +#include <utilstrencodings.h> // for GetTime() #include <stdlib.h> #include <limits> diff --git a/src/random.h b/src/random.h index c60ab36179..ea88670a37 100644 --- a/src/random.h +++ b/src/random.h @@ -1,14 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_RANDOM_H #define BITCOIN_RANDOM_H -#include "crypto/chacha20.h" -#include "crypto/common.h" -#include "uint256.h" +#include <crypto/chacha20.h> +#include <crypto/common.h> +#include <uint256.h> #include <stdint.h> @@ -128,7 +128,7 @@ public: * sure that the underlying OS APIs for all platforms support the number. * (many cap out at 256 bytes). */ -static const ssize_t NUM_OS_RANDOM_BYTES = 32; +static const int NUM_OS_RANDOM_BYTES = 32; /** Get 32 bytes of system entropy. Do not use this in application code: use * GetStrongRandBytes instead. diff --git a/src/rest.cpp b/src/rest.cpp index 0b2c843d5f..0c93ce020e 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -1,22 +1,22 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" -#include "chainparams.h" -#include "core_io.h" -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "validation.h" -#include "httpserver.h" -#include "rpc/blockchain.h" -#include "rpc/server.h" -#include "streams.h" -#include "sync.h" -#include "txmempool.h" -#include "utilstrencodings.h" -#include "version.h" +#include <chain.h> +#include <chainparams.h> +#include <core_io.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <validation.h> +#include <httpserver.h> +#include <rpc/blockchain.h> +#include <rpc/server.h> +#include <streams.h> +#include <sync.h> +#include <txmempool.h> +#include <utilstrencodings.h> +#include <version.h> #include <boost/algorithm/string.hpp> @@ -178,8 +178,11 @@ static bool rest_headers(HTTPRequest* req, } case RF_JSON: { UniValue jsonHeaders(UniValue::VARR); - for (const CBlockIndex *pindex : headers) { - jsonHeaders.push_back(blockheaderToJSON(pindex)); + { + LOCK(cs_main); + for (const CBlockIndex *pindex : headers) { + jsonHeaders.push_back(blockheaderToJSON(pindex)); + } } std::string strJSON = jsonHeaders.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); @@ -239,7 +242,11 @@ static bool rest_block(HTTPRequest* req, } case RF_JSON: { - UniValue objBlock = blockToJSON(block, pblockindex, showTxDetails); + UniValue objBlock; + { + LOCK(cs_main); + objBlock = blockToJSON(block, pblockindex, showTxDetails); + } std::string strJSON = objBlock.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -409,10 +416,8 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) if (uriParts.size() > 0) { - //inputs is sent over URI scheme (/rest/getutxos/checkmempool/txid1-n/txid2-n/...) - if (uriParts.size() > 0 && uriParts[0] == "checkmempool") - fCheckMemPool = true; + if (uriParts[0] == "checkmempool") fCheckMemPool = true; for (size_t i = (fCheckMemPool) ? 1 : 0; i < uriParts.size(); i++) { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 68af376f35..1b2c71c4a4 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,30 +1,30 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/blockchain.h" - -#include "amount.h" -#include "chain.h" -#include "chainparams.h" -#include "checkpoints.h" -#include "coins.h" -#include "consensus/validation.h" -#include "validation.h" -#include "core_io.h" -#include "policy/feerate.h" -#include "policy/policy.h" -#include "primitives/transaction.h" -#include "rpc/server.h" -#include "streams.h" -#include "sync.h" -#include "txdb.h" -#include "txmempool.h" -#include "util.h" -#include "utilstrencodings.h" -#include "hash.h" -#include "warnings.h" +#include <rpc/blockchain.h> + +#include <amount.h> +#include <chain.h> +#include <chainparams.h> +#include <checkpoints.h> +#include <coins.h> +#include <consensus/validation.h> +#include <validation.h> +#include <core_io.h> +#include <policy/feerate.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <rpc/server.h> +#include <streams.h> +#include <sync.h> +#include <txdb.h> +#include <txmempool.h> +#include <util.h> +#include <utilstrencodings.h> +#include <hash.h> +#include <warnings.h> #include <stdint.h> @@ -47,18 +47,20 @@ static CUpdatedBlock latestblock; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -double GetDifficulty(const CBlockIndex* blockindex) +/* Calculate the difficulty for a given block index, + * or the block index of the given chain. + */ +double GetDifficulty(const CChain& chain, const CBlockIndex* blockindex) { if (blockindex == nullptr) { - if (chainActive.Tip() == nullptr) + if (chain.Tip() == nullptr) return 1.0; else - blockindex = chainActive.Tip(); + blockindex = chain.Tip(); } int nShift = (blockindex->nBits >> 24) & 0xff; - double dDiff = (double)0x0000ffff / (double)(blockindex->nBits & 0x00ffffff); @@ -76,8 +78,14 @@ double GetDifficulty(const CBlockIndex* blockindex) return dDiff; } +double GetDifficulty(const CBlockIndex* blockindex) +{ + return GetDifficulty(chainActive, blockindex); +} + UniValue blockheaderToJSON(const CBlockIndex* blockindex) { + AssertLockHeld(cs_main); UniValue result(UniValue::VOBJ); result.push_back(Pair("hash", blockindex->GetBlockHash().GetHex())); int confirmations = -1; @@ -106,6 +114,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex) UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails) { + AssertLockHeld(cs_main); UniValue result(UniValue::VOBJ); result.push_back(Pair("hash", blockindex->GetBlockHash().GetHex())); int confirmations = -1; @@ -926,7 +935,7 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) CCoinsStats stats; FlushStateToDisk(); - if (GetUTXOStats(pcoinsdbview, stats)) { + if (GetUTXOStats(pcoinsdbview.get(), stats)) { ret.push_back(Pair("height", (int64_t)stats.nHeight)); ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); @@ -994,7 +1003,7 @@ UniValue gettxout(const JSONRPCRequest& request) Coin coin; if (fMempool) { LOCK(mempool.cs); - CCoinsViewMemPool view(pcoinsTip, mempool); + CCoinsViewMemPool view(pcoinsTip.get(), mempool); if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { return NullUniValue; } @@ -1046,7 +1055,7 @@ UniValue verifychain(const JSONRPCRequest& request) if (!request.params[1].isNull()) nCheckDepth = request.params[1].get_int(); - return CVerifyDB().VerifyDB(Params(), pcoinsTip, nCheckLevel, nCheckDepth); + return CVerifyDB().VerifyDB(Params(), pcoinsTip.get(), nCheckLevel, nCheckDepth); } /** Implementation of IsSuperMajority with better feedback */ @@ -1111,13 +1120,13 @@ static UniValue BIP9SoftForkDesc(const Consensus::Params& consensusParams, Conse return rv; } -void BIP9SoftForkDescPushBack(UniValue& bip9_softforks, const std::string &name, const Consensus::Params& consensusParams, Consensus::DeploymentPos id) +void BIP9SoftForkDescPushBack(UniValue& bip9_softforks, const Consensus::Params& consensusParams, Consensus::DeploymentPos id) { // Deployments with timeout value of 0 are hidden. // A timeout value of 0 guarantees a softfork will never be activated. // This is used when softfork codes are merged without specifying the deployment schedule. if (consensusParams.vDeployments[id].nTimeout > 0) - bip9_softforks.push_back(Pair(name, BIP9SoftForkDesc(consensusParams, id))); + bip9_softforks.push_back(Pair(VersionBitsDeploymentInfo[id].name, BIP9SoftForkDesc(consensusParams, id))); } UniValue getblockchaininfo(const JSONRPCRequest& request) @@ -1128,45 +1137,46 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) "Returns an object containing various state info regarding blockchain processing.\n" "\nResult:\n" "{\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" - " \"blocks\": xxxxxx, (numeric) the current number of blocks processed in the server\n" - " \"headers\": xxxxxx, (numeric) the current number of headers we have validated\n" - " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" - " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" - " \"mediantime\": xxxxxx, (numeric) median time for the current best block\n" + " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"blocks\": xxxxxx, (numeric) the current number of blocks processed in the server\n" + " \"headers\": xxxxxx, (numeric) the current number of headers we have validated\n" + " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" + " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" + " \"mediantime\": xxxxxx, (numeric) median time for the current best block\n" " \"verificationprogress\": xxxx, (numeric) estimate of verification progress [0..1]\n" - " \"chainwork\": \"xxxx\" (string) total amount of work in active chain, in hexadecimal\n" - " \"size_on_disk\": xxxxxx, (numeric) the estimated size of the block and undo files on disk\n" - " \"pruned\": xx, (boolean) if the blocks are subject to pruning\n" - " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored (only present if pruning is enabled)\n" - " \"automatic_pruning\": xx, (boolean) whether automatic pruning is enabled (only present if pruning is enabled)\n" + " \"initialblockdownload\": xxxx, (bool) (debug information) estimate of whether this node is in Initial Block Download mode.\n" + " \"chainwork\": \"xxxx\" (string) total amount of work in active chain, in hexadecimal\n" + " \"size_on_disk\": xxxxxx, (numeric) the estimated size of the block and undo files on disk\n" + " \"pruned\": xx, (boolean) if the blocks are subject to pruning\n" + " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored (only present if pruning is enabled)\n" + " \"automatic_pruning\": xx, (boolean) whether automatic pruning is enabled (only present if pruning is enabled)\n" " \"prune_target_size\": xxxxxx, (numeric) the target size used by pruning (only present if automatic pruning is enabled)\n" - " \"softforks\": [ (array) status of softforks in progress\n" + " \"softforks\": [ (array) status of softforks in progress\n" " {\n" - " \"id\": \"xxxx\", (string) name of softfork\n" - " \"version\": xx, (numeric) block version\n" - " \"reject\": { (object) progress toward rejecting pre-softfork blocks\n" - " \"status\": xx, (boolean) true if threshold reached\n" + " \"id\": \"xxxx\", (string) name of softfork\n" + " \"version\": xx, (numeric) block version\n" + " \"reject\": { (object) progress toward rejecting pre-softfork blocks\n" + " \"status\": xx, (boolean) true if threshold reached\n" " },\n" " }, ...\n" " ],\n" - " \"bip9_softforks\": { (object) status of BIP9 softforks in progress\n" - " \"xxxx\" : { (string) name of the softfork\n" - " \"status\": \"xxxx\", (string) one of \"defined\", \"started\", \"locked_in\", \"active\", \"failed\"\n" - " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" - " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" - " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" - " \"since\": xx, (numeric) height of the first block to which the status applies\n" - " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" - " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" - " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" - " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" - " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" - " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" + " \"bip9_softforks\": { (object) status of BIP9 softforks in progress\n" + " \"xxxx\" : { (string) name of the softfork\n" + " \"status\": \"xxxx\", (string) one of \"defined\", \"started\", \"locked_in\", \"active\", \"failed\"\n" + " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" + " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" + " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" + " \"since\": xx, (numeric) height of the first block to which the status applies\n" + " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" + " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" + " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" + " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" + " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" + " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" " }\n" " }\n" " }\n" - " \"warnings\" : \"...\", (string) any network and blockchain warnings.\n" + " \"warnings\" : \"...\", (string) any network and blockchain warnings.\n" "}\n" "\nExamples:\n" + HelpExampleCli("getblockchaininfo", "") @@ -1183,6 +1193,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) obj.push_back(Pair("difficulty", (double)GetDifficulty())); obj.push_back(Pair("mediantime", (int64_t)chainActive.Tip()->GetMedianTimePast())); obj.push_back(Pair("verificationprogress", GuessVerificationProgress(Params().TxData(), chainActive.Tip()))); + obj.push_back(Pair("initialblockdownload", IsInitialBlockDownload())); obj.push_back(Pair("chainwork", chainActive.Tip()->nChainWork.GetHex())); obj.push_back(Pair("size_on_disk", CalculateCurrentUsage())); obj.push_back(Pair("pruned", fPruneMode)); @@ -1210,8 +1221,9 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) softforks.push_back(SoftForkDesc("bip34", 2, tip, consensusParams)); softforks.push_back(SoftForkDesc("bip66", 3, tip, consensusParams)); softforks.push_back(SoftForkDesc("bip65", 4, tip, consensusParams)); - BIP9SoftForkDescPushBack(bip9_softforks, "csv", consensusParams, Consensus::DEPLOYMENT_CSV); - BIP9SoftForkDescPushBack(bip9_softforks, "segwit", consensusParams, Consensus::DEPLOYMENT_SEGWIT); + for (int pos = Consensus::DEPLOYMENT_CSV; pos != Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++pos) { + BIP9SoftForkDescPushBack(bip9_softforks, consensusParams, static_cast<Consensus::DeploymentPos>(pos)); + } obj.push_back(Pair("softforks", softforks)); obj.push_back(Pair("bip9_softforks", bip9_softforks)); @@ -1345,7 +1357,8 @@ UniValue mempoolInfoToJSON() ret.push_back(Pair("usage", (int64_t) mempool.DynamicMemoryUsage())); size_t maxmempool = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; ret.push_back(Pair("maxmempool", (int64_t) maxmempool)); - ret.push_back(Pair("mempoolminfee", ValueFromAmount(mempool.GetMinFee(maxmempool).GetFeePerK()))); + ret.push_back(Pair("mempoolminfee", ValueFromAmount(std::max(mempool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK()))); + ret.push_back(Pair("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK()))); return ret; } @@ -1362,7 +1375,8 @@ UniValue getmempoolinfo(const JSONRPCRequest& request) " \"bytes\": xxxxx, (numeric) Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted\n" " \"usage\": xxxxx, (numeric) Total memory usage for the mempool\n" " \"maxmempool\": xxxxx, (numeric) Maximum memory usage for the mempool\n" - " \"mempoolminfee\": xxxxx (numeric) Minimum fee rate in " + CURRENCY_UNIT + "/kB for tx to be accepted\n" + " \"mempoolminfee\": xxxxx (numeric) Minimum fee rate in " + CURRENCY_UNIT + "/kB for tx to be accepted. Is the maximum of minrelaytxfee and minimum mempool fee\n" + " \"minrelaytxfee\": xxxxx (numeric) Current minimum relay fee for transactions\n" "}\n" "\nExamples:\n" + HelpExampleCli("getmempoolinfo", "") diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index f54f24e2a7..f63e970bdd 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -1,17 +1,15 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/client.h" -#include "rpc/protocol.h" -#include "util.h" +#include <rpc/client.h> +#include <rpc/protocol.h> +#include <util.h> #include <set> #include <stdint.h> -#include <univalue.h> - class CRPCConvertParam { public: @@ -93,11 +91,13 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createrawtransaction", 1, "outputs" }, { "createrawtransaction", 2, "locktime" }, { "createrawtransaction", 3, "replaceable" }, + { "decoderawtransaction", 1, "iswitness" }, { "signrawtransaction", 1, "prevtxs" }, { "signrawtransaction", 2, "privkeys" }, { "sendrawtransaction", 1, "allowhighfees" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, + { "fundrawtransaction", 2, "iswitness" }, { "gettxout", 1, "n" }, { "gettxout", 2, "include_mempool" }, { "gettxoutproof", 0, "txids" }, @@ -141,6 +141,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "echojson", 7, "arg7" }, { "echojson", 8, "arg8" }, { "echojson", 9, "arg9" }, + { "rescanblockchain", 0, "start_height"}, + { "rescanblockchain", 1, "stop_height"}, }; class CRPCConvertTable diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index f79439f038..c22d0ac377 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -1,36 +1,34 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" -#include "amount.h" -#include "chain.h" -#include "chainparams.h" -#include "consensus/consensus.h" -#include "consensus/params.h" -#include "consensus/validation.h" -#include "core_io.h" -#include "init.h" -#include "validation.h" -#include "miner.h" -#include "net.h" -#include "policy/fees.h" -#include "pow.h" -#include "rpc/blockchain.h" -#include "rpc/mining.h" -#include "rpc/server.h" -#include "txmempool.h" -#include "util.h" -#include "utilstrencodings.h" -#include "validationinterface.h" -#include "warnings.h" +#include <base58.h> +#include <amount.h> +#include <chain.h> +#include <chainparams.h> +#include <consensus/consensus.h> +#include <consensus/params.h> +#include <consensus/validation.h> +#include <core_io.h> +#include <init.h> +#include <validation.h> +#include <miner.h> +#include <net.h> +#include <policy/fees.h> +#include <pow.h> +#include <rpc/blockchain.h> +#include <rpc/mining.h> +#include <rpc/server.h> +#include <txmempool.h> +#include <util.h> +#include <utilstrencodings.h> +#include <validationinterface.h> +#include <warnings.h> #include <memory> #include <stdint.h> -#include <univalue.h> - unsigned int ParseConfirmTarget(const UniValue& value) { int target = value.get_int(); @@ -455,7 +453,7 @@ UniValue getblocktemplate(const JSONRPCRequest& request) { // Wait to respond until either the best block changes, OR a minute has passed and there are more transactions uint256 hashWatchedChain; - boost::system_time checktxtime; + std::chrono::steady_clock::time_point checktxtime; unsigned int nTransactionsUpdatedLastLP; if (lpval.isStr()) @@ -476,17 +474,17 @@ UniValue getblocktemplate(const JSONRPCRequest& request) // Release the wallet and main lock while waiting LEAVE_CRITICAL_SECTION(cs_main); { - checktxtime = boost::get_system_time() + boost::posix_time::minutes(1); + checktxtime = std::chrono::steady_clock::now() + std::chrono::minutes(1); - boost::unique_lock<boost::mutex> lock(csBestBlock); + WaitableLock lock(csBestBlock); while (chainActive.Tip()->GetBlockHash() == hashWatchedChain && IsRPCRunning()) { - if (!cvBlockChange.timed_wait(lock, checktxtime)) + if (cvBlockChange.wait_until(lock, checktxtime) == std::cv_status::timeout) { // Timeout: Check transactions for update if (mempool.GetTransactionsUpdated() != nTransactionsUpdatedLastLP) break; - checktxtime += boost::posix_time::seconds(10); + checktxtime += std::chrono::seconds(10); } } } diff --git a/src/rpc/mining.h b/src/rpc/mining.h index 868d7002b5..8d46273159 100644 --- a/src/rpc/mining.h +++ b/src/rpc/mining.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_RPC_MINING_H #define BITCOIN_RPC_MINING_H -#include "script/script.h" +#include <script/script.h> #include <univalue.h> diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 521b49e2a7..95a27d474b 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -1,29 +1,29 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" -#include "chain.h" -#include "clientversion.h" -#include "core_io.h" -#include "crypto/ripemd160.h" -#include "init.h" -#include "validation.h" -#include "httpserver.h" -#include "net.h" -#include "netbase.h" -#include "rpc/blockchain.h" -#include "rpc/server.h" -#include "timedata.h" -#include "util.h" -#include "utilstrencodings.h" +#include <base58.h> +#include <chain.h> +#include <clientversion.h> +#include <core_io.h> +#include <crypto/ripemd160.h> +#include <init.h> +#include <validation.h> +#include <httpserver.h> +#include <net.h> +#include <netbase.h> +#include <rpc/blockchain.h> +#include <rpc/server.h> +#include <timedata.h> +#include <util.h> +#include <utilstrencodings.h> #ifdef ENABLE_WALLET -#include "wallet/rpcwallet.h" -#include "wallet/wallet.h" -#include "wallet/walletdb.h" +#include <wallet/rpcwallet.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> #endif -#include "warnings.h" +#include <warnings.h> #include <stdint.h> #ifdef HAVE_MALLOC_INFO @@ -40,6 +40,46 @@ public: explicit DescribeAddressVisitor(CWallet *_pwallet) : pwallet(_pwallet) {} + void ProcessSubScript(const CScript& subscript, UniValue& obj, bool include_addresses = false) const + { + // Always present: script type and redeemscript + txnouttype which_type; + std::vector<std::vector<unsigned char>> solutions_data; + Solver(subscript, which_type, solutions_data); + obj.pushKV("script", GetTxnOutputType(which_type)); + obj.pushKV("hex", HexStr(subscript.begin(), subscript.end())); + + CTxDestination embedded; + UniValue a(UniValue::VARR); + if (ExtractDestination(subscript, embedded)) { + // Only when the script corresponds to an address. + UniValue subobj = boost::apply_visitor(*this, embedded); + subobj.pushKV("address", EncodeDestination(embedded)); + subobj.pushKV("scriptPubKey", HexStr(subscript.begin(), subscript.end())); + // Always report the pubkey at the top level, so that `getnewaddress()['pubkey']` always works. + if (subobj.exists("pubkey")) obj.pushKV("pubkey", subobj["pubkey"]); + obj.pushKV("embedded", std::move(subobj)); + if (include_addresses) a.push_back(EncodeDestination(embedded)); + } else if (which_type == TX_MULTISIG) { + // Also report some information on multisig scripts (which do not have a corresponding address). + // TODO: abstract out the common functionality between this logic and ExtractDestinations. + obj.pushKV("sigsrequired", solutions_data[0][0]); + UniValue pubkeys(UniValue::VARR); + for (size_t i = 1; i < solutions_data.size() - 1; ++i) { + CPubKey key(solutions_data[i].begin(), solutions_data[i].end()); + if (include_addresses) a.push_back(EncodeDestination(key.GetID())); + pubkeys.push_back(HexStr(key.begin(), key.end())); + } + obj.pushKV("pubkeys", std::move(pubkeys)); + } + + // The "addresses" field is confusing because it refers to public keys using their P2PKH address. + // For that reason, only add the 'addresses' field when needed for backward compatibility. New applications + // can use the 'embedded'->'address' field for P2SH or P2WSH wrapped addresses, and 'pubkeys' for + // inspecting multisig participants. + if (include_addresses) obj.pushKV("addresses", std::move(a)); + } + UniValue operator()(const CNoDestination &dest) const { return UniValue(UniValue::VOBJ); } UniValue operator()(const CKeyID &keyID) const { @@ -60,19 +100,7 @@ public: obj.push_back(Pair("isscript", true)); obj.push_back(Pair("iswitness", false)); if (pwallet && pwallet->GetCScript(scriptID, subscript)) { - std::vector<CTxDestination> addresses; - txnouttype whichType; - int nRequired; - ExtractDestinations(subscript, whichType, addresses, nRequired); - obj.push_back(Pair("script", GetTxnOutputType(whichType))); - obj.push_back(Pair("hex", HexStr(subscript.begin(), subscript.end()))); - UniValue a(UniValue::VARR); - for (const CTxDestination& addr : addresses) { - a.push_back(EncodeDestination(addr)); - } - obj.push_back(Pair("addresses", a)); - if (whichType == TX_MULTISIG) - obj.push_back(Pair("sigsrequired", nRequired)); + ProcessSubScript(subscript, obj, true); } return obj; } @@ -103,7 +131,7 @@ public: uint160 hash; hasher.Write(id.begin(), 32).Finalize(hash.begin()); if (pwallet && pwallet->GetCScript(CScriptID(hash), subscript)) { - obj.push_back(Pair("hex", HexStr(subscript.begin(), subscript.end()))); + ProcessSubScript(subscript, obj); } return obj; } @@ -131,23 +159,32 @@ UniValue validateaddress(const JSONRPCRequest& request) "\nResult:\n" "{\n" " \"isvalid\" : true|false, (boolean) If the address is valid or not. If not, this is the only property returned.\n" - " \"address\" : \"address\", (string) The bitcoin address validated\n" + " \"address\" : \"address\", (string) The bitcoin address validated\n" " \"scriptPubKey\" : \"hex\", (string) The hex encoded scriptPubKey generated by the address\n" " \"ismine\" : true|false, (boolean) If the address is yours or not\n" " \"iswatchonly\" : true|false, (boolean) If the address is watchonly\n" - " \"isscript\" : true|false, (boolean) If the key is a script\n" - " \"script\" : \"type\" (string, optional) The output script type. Possible types: nonstandard, pubkey, pubkeyhash, scripthash, multisig, nulldata, witness_v0_keyhash, witness_v0_scripthash\n" - " \"hex\" : \"hex\", (string, optional) The redeemscript for the p2sh address\n" - " \"addresses\" (string, optional) Array of addresses associated with the known redeemscript\n" + " \"isscript\" : true|false, (boolean, optional) If the address is P2SH or P2WSH. Not included for unknown witness types.\n" + " \"iswitness\" : true|false, (boolean) If the address is P2WPKH, P2WSH, or an unknown witness version\n" + " \"witness_version\" : version (number, optional) For all witness output types, gives the version number.\n" + " \"witness_program\" : \"hex\" (string, optional) For all witness output types, gives the script or key hash present in the address.\n" + " \"script\" : \"type\" (string, optional) The output script type. Only if \"isscript\" is true and the redeemscript is known. Possible types: nonstandard, pubkey, pubkeyhash, scripthash, multisig, nulldata, witness_v0_keyhash, witness_v0_scripthash, witness_unknown\n" + " \"hex\" : \"hex\", (string, optional) The redeemscript for the P2SH or P2WSH address\n" + " \"addresses\" (string, optional) Array of addresses associated with the known redeemscript (only if \"iswitness\" is false). This field is superseded by the \"pubkeys\" field and the address inside \"embedded\".\n" " [\n" " \"address\"\n" " ,...\n" " ]\n" - " \"sigsrequired\" : xxxxx (numeric, optional) Number of signatures required to spend multisig output\n" - " \"pubkey\" : \"publickeyhex\", (string) The hex value of the raw public key\n" + " \"pubkeys\" (string, optional) Array of pubkeys associated with the known redeemscript (only if \"script\" is \"multisig\")\n" + " [\n" + " \"pubkey\"\n" + " ,...\n" + " ]\n" + " \"sigsrequired\" : xxxxx (numeric, optional) Number of signatures required to spend multisig output (only if \"script\" is \"multisig\")\n" + " \"pubkey\" : \"publickeyhex\", (string, optional) The hex value of the raw public key, for single-key addresses (possibly embedded in P2SH or P2WSH)\n" + " \"embedded\" : {...}, (object, optional) information about the address embedded in P2SH or P2WSH, if relevant and known. It includes all validateaddress output fields for the embedded address, excluding \"isvalid\", metadata (\"timestamp\", \"hdkeypath\", \"hdmasterkeyid\") and relation to the wallet (\"ismine\", \"iswatchonly\", \"account\").\n" " \"iscompressed\" : true|false, (boolean) If the address is compressed\n" " \"account\" : \"account\" (string) DEPRECATED. The account associated with the address, \"\" is the default account\n" - " \"timestamp\" : timestamp, (number, optional) The creation time of the key if available in seconds since epoch (Jan 1 1970 GMT)\n" + " \"timestamp\" : timestamp, (number, optional) The creation time of the key if available in seconds since epoch (Jan 1 1970 GMT)\n" " \"hdkeypath\" : \"keypath\" (string, optional) The HD keypath if the key is HD and available\n" " \"hdmasterkeyid\" : \"<hash160>\" (string, optional) The Hash160 of the HD master pubkey\n" "}\n" @@ -187,17 +224,25 @@ UniValue validateaddress(const JSONRPCRequest& request) ret.push_back(Pair("account", pwallet->mapAddressBook[dest].name)); } if (pwallet) { - const auto& meta = pwallet->mapKeyMetadata; - const CKeyID *keyID = boost::get<CKeyID>(&dest); - auto it = keyID ? meta.find(*keyID) : meta.end(); - if (it == meta.end()) { - it = meta.find(CScriptID(scriptPubKey)); + const CKeyMetadata* meta = nullptr; + CKeyID key_id = GetKeyForDestination(*pwallet, dest); + if (!key_id.IsNull()) { + auto it = pwallet->mapKeyMetadata.find(key_id); + if (it != pwallet->mapKeyMetadata.end()) { + meta = &it->second; + } + } + if (!meta) { + auto it = pwallet->m_script_metadata.find(CScriptID(scriptPubKey)); + if (it != pwallet->m_script_metadata.end()) { + meta = &it->second; + } } - if (it != meta.end()) { - ret.push_back(Pair("timestamp", it->second.nCreateTime)); - if (!it->second.hdKeypath.empty()) { - ret.push_back(Pair("hdkeypath", it->second.hdKeypath)); - ret.push_back(Pair("hdmasterkeyid", it->second.hdMasterKeyID.GetHex())); + if (meta) { + ret.push_back(Pair("timestamp", meta->nCreateTime)); + if (!meta->hdKeypath.empty()) { + ret.push_back(Pair("hdkeypath", meta->hdKeypath)); + ret.push_back(Pair("hdmasterkeyid", meta->hdMasterKeyID.GetHex())); } } } @@ -235,12 +280,12 @@ CScript _createmultisig_redeemScript(CWallet * const pwallet, const UniValue& pa // Case 1: Bitcoin address and we have full public key: CTxDestination dest = DecodeDestination(ks); if (pwallet && IsValidDestination(dest)) { - const CKeyID *keyID = boost::get<CKeyID>(&dest); - if (!keyID) { + CKeyID key = GetKeyForDestination(*pwallet, dest); + if (key.IsNull()) { throw std::runtime_error(strprintf("%s does not refer to a key", ks)); } CPubKey vchPubKey; - if (!pwallet->GetPubKey(*keyID, vchPubKey)) { + if (!pwallet->GetPubKey(key, vchPubKey)) { throw std::runtime_error(strprintf("no full public key for address %s", ks)); } if (!vchPubKey.IsFullyValid()) @@ -533,6 +578,9 @@ uint32_t getCategoryMask(UniValue cats) { if (!GetLogCategory(&flag, &cat)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown logging category " + cat); } + if (flag == BCLog::NONE) { + return 0; + } mask |= flag; } return mask; @@ -542,16 +590,32 @@ UniValue logging(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() > 2) { throw std::runtime_error( - "logging [include,...] <exclude>\n" + "logging ( <include> <exclude> )\n" "Gets and sets the logging configuration.\n" - "When called without an argument, returns the list of categories that are currently being debug logged.\n" - "When called with arguments, adds or removes categories from debug logging.\n" + "When called without an argument, returns the list of categories with status that are currently being debug logged or not.\n" + "When called with arguments, adds or removes categories from debug logging and return the lists above.\n" + "The arguments are evaluated in order \"include\", \"exclude\".\n" + "If an item is both included and excluded, it will thus end up being excluded.\n" "The valid logging categories are: " + ListLogCategories() + "\n" - "libevent logging is configured on startup and cannot be modified by this RPC during runtime." - "Arguments:\n" - "1. \"include\" (array of strings) add debug logging for these categories.\n" - "2. \"exclude\" (array of strings) remove debug logging for these categories.\n" - "\nResult: <categories> (string): a list of the logging categories that are active.\n" + "In addition, the following are available as category names with special meanings:\n" + " - \"all\", \"1\" : represent all logging categories.\n" + " - \"none\", \"0\" : even if other logging categories are specified, ignore all of them.\n" + "\nArguments:\n" + "1. \"include\" (array of strings, optional) A json array of categories to add debug logging\n" + " [\n" + " \"category\" (string) the valid logging category\n" + " ,...\n" + " ]\n" + "2. \"exclude\" (array of strings, optional) A json array of categories to remove debug logging\n" + " [\n" + " \"category\" (string) the valid logging category\n" + " ,...\n" + " ]\n" + "\nResult:\n" + "{ (json object where keys are the logging categories, and values indicates its status\n" + " \"category\": 0|1, (numeric) if being debug logged or not. 0:inactive, 1:active\n" + " ...\n" + "}\n" "\nExamples:\n" + HelpExampleCli("logging", "\"[\\\"all\\\"]\" \"[\\\"http\\\"]\"") + HelpExampleRpc("logging", "[\"all\"], \"[libevent]\"") @@ -608,6 +672,7 @@ static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- { "control", "getmemoryinfo", &getmemoryinfo, {"mode"} }, + { "control", "logging", &logging, {"include", "exclude"}}, { "util", "validateaddress", &validateaddress, {"address"} }, /* uses wallet if enabled */ { "util", "createmultisig", &createmultisig, {"nrequired","keys"} }, { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, @@ -617,7 +682,6 @@ static const CRPCCommand commands[] = { "hidden", "setmocktime", &setmocktime, {"timestamp"}}, { "hidden", "echo", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, { "hidden", "echojson", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, - { "hidden", "logging", &logging, {"include", "exclude"}}, }; void RegisterMiscRPCCommands(CRPCTable &t) diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index a3d3df26a3..a81b651981 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -1,25 +1,25 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/server.h" - -#include "chainparams.h" -#include "clientversion.h" -#include "core_io.h" -#include "validation.h" -#include "net.h" -#include "net_processing.h" -#include "netbase.h" -#include "policy/policy.h" -#include "protocol.h" -#include "sync.h" -#include "timedata.h" -#include "ui_interface.h" -#include "util.h" -#include "utilstrencodings.h" -#include "version.h" -#include "warnings.h" +#include <rpc/server.h> + +#include <chainparams.h> +#include <clientversion.h> +#include <core_io.h> +#include <validation.h> +#include <net.h> +#include <net_processing.h> +#include <netbase.h> +#include <policy/policy.h> +#include <rpc/protocol.h> +#include <sync.h> +#include <timedata.h> +#include <ui_interface.h> +#include <util.h> +#include <utilstrencodings.h> +#include <version.h> +#include <warnings.h> #include <univalue.h> @@ -92,7 +92,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) " \"version\": v, (numeric) The peer version, such as 7001\n" " \"subver\": \"/Satoshi:0.8.5/\", (string) The string version\n" " \"inbound\": true|false, (boolean) Inbound (true) or Outbound (false)\n" - " \"addnode\": true|false, (boolean) Whether connection was due to addnode and is using an addnode slot\n" + " \"addnode\": true|false, (boolean) Whether connection was due to addnode/-connect or if it was an automatic/inbound connection\n" " \"startingheight\": n, (numeric) The starting height (block) of the peer\n" " \"banscore\": n, (numeric) The ban score\n" " \"synced_headers\": n, (numeric) The last header we have in common with this peer\n" @@ -156,7 +156,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) // their ver message. obj.push_back(Pair("subver", stats.cleanSubVer)); obj.push_back(Pair("inbound", stats.fInbound)); - obj.push_back(Pair("addnode", stats.fAddnode)); + obj.push_back(Pair("addnode", stats.m_manual_connection)); obj.push_back(Pair("startingheight", stats.nStartingHeight)); if (fStateStats) { obj.push_back(Pair("banscore", statestats.nMisbehavior)); @@ -201,6 +201,8 @@ UniValue addnode(const JSONRPCRequest& request) "addnode \"node\" \"add|remove|onetry\"\n" "\nAttempts to add or remove a node from the addnode list.\n" "Or try a connection to a node once.\n" + "Nodes added using addnode (or -connect) are protected from DoS disconnection and are not required to be\n" + "full nodes/support SegWit as other outbound peers are (though such peers will not be synced from).\n" "\nArguments:\n" "1. \"node\" (string, required) The node (see getpeerinfo for nodes)\n" "2. \"command\" (string, required) 'add' to add a node to the list, 'remove' to remove a node from the list, 'onetry' to try a connection to the node once\n" @@ -217,7 +219,7 @@ UniValue addnode(const JSONRPCRequest& request) if (strCommand == "onetry") { CAddress addr; - g_connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str()); + g_connman->OpenNetworkConnection(addr, false, nullptr, strNode.c_str(), false, false, true); return NullUniValue; } @@ -569,11 +571,11 @@ UniValue listbanned(const JSONRPCRequest& request) g_connman->GetBanned(banMap); UniValue bannedAddresses(UniValue::VARR); - for (banmap_t::iterator it = banMap.begin(); it != banMap.end(); it++) + for (const auto& entry : banMap) { - CBanEntry banEntry = (*it).second; + const CBanEntry& banEntry = entry.second; UniValue rec(UniValue::VOBJ); - rec.push_back(Pair("address", (*it).first.ToString())); + rec.push_back(Pair("address", entry.first.ToString())); rec.push_back(Pair("banned_until", banEntry.nBanUntil)); rec.push_back(Pair("ban_created", banEntry.nCreateTime)); rec.push_back(Pair("ban_reason", banEntry.banReasonToString())); diff --git a/src/rpc/protocol.cpp b/src/rpc/protocol.cpp index 1f4ae75b18..ddc1bb6232 100644 --- a/src/rpc/protocol.cpp +++ b/src/rpc/protocol.cpp @@ -1,18 +1,17 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/protocol.h" +#include <rpc/protocol.h> -#include "random.h" -#include "tinyformat.h" -#include "util.h" -#include "utilstrencodings.h" -#include "utiltime.h" -#include "version.h" +#include <random.h> +#include <tinyformat.h> +#include <util.h> +#include <utilstrencodings.h> +#include <utiltime.h> +#include <version.h> -#include <stdint.h> #include <fstream> /** diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index cb668f3db9..00b92f1956 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -1,12 +1,12 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_RPCPROTOCOL_H #define BITCOIN_RPCPROTOCOL_H -#include "fs.h" +#include <fs.h> #include <list> #include <map> diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index d860dbc244..82399e1567 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1,35 +1,37 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" -#include "chain.h" -#include "coins.h" -#include "consensus/validation.h" -#include "core_io.h" -#include "init.h" -#include "keystore.h" -#include "validation.h" -#include "merkleblock.h" -#include "net.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "primitives/transaction.h" -#include "rpc/safemode.h" -#include "rpc/server.h" -#include "script/script.h" -#include "script/script_error.h" -#include "script/sign.h" -#include "script/standard.h" -#include "txmempool.h" -#include "uint256.h" -#include "utilstrencodings.h" +#include <base58.h> +#include <chain.h> +#include <coins.h> +#include <consensus/validation.h> +#include <core_io.h> +#include <init.h> +#include <keystore.h> +#include <validation.h> +#include <validationinterface.h> +#include <merkleblock.h> +#include <net.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <primitives/transaction.h> +#include <rpc/safemode.h> +#include <rpc/server.h> +#include <script/script.h> +#include <script/script_error.h> +#include <script/sign.h> +#include <script/standard.h> +#include <txmempool.h> +#include <uint256.h> +#include <utilstrencodings.h> #ifdef ENABLE_WALLET -#include "wallet/rpcwallet.h" -#include "wallet/wallet.h" +#include <wallet/rpcwallet.h> +#include <wallet/wallet.h> #endif +#include <future> #include <stdint.h> #include <univalue.h> @@ -62,12 +64,15 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) UniValue getrawtransaction(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) throw std::runtime_error( - "getrawtransaction \"txid\" ( verbose )\n" + "getrawtransaction \"txid\" ( verbose \"blockhash\" )\n" "\nNOTE: By default this function only works for mempool transactions. If the -txindex option is\n" - "enabled, it also works for blockchain transactions.\n" + "enabled, it also works for blockchain transactions. If the block which contains the transaction\n" + "is known, its hash can be provided even for nodes without -txindex. Note that if a blockhash is\n" + "provided, only that block will be searched and if the transaction is in the mempool or other\n" + "blocks, or if this node does not have the given block available, the transaction will not be found.\n" "DEPRECATED: for now, it also works for transactions with unspent outputs.\n" "\nReturn the raw transaction data.\n" @@ -76,13 +81,15 @@ UniValue getrawtransaction(const JSONRPCRequest& request) "\nArguments:\n" "1. \"txid\" (string, required) The transaction id\n" - "2. verbose (bool, optional, default=false) If false, return a string, otherwise return a json object\n" + "2. verbose (bool, optional, default=false) If false, return a string, otherwise return a json object\n" + "3. \"blockhash\" (string, optional) The block in which to look for the transaction\n" "\nResult (if verbose is not set or set to false):\n" "\"data\" (string) The serialized, hex-encoded data for 'txid'\n" "\nResult (if verbose is set to true):\n" "{\n" + " \"in_active_chain\": b, (bool) Whether specified block is in the active chain or not (only present with explicit \"blockhash\" argument)\n" " \"hex\" : \"data\", (string) The serialized, hex-encoded data for 'txid'\n" " \"txid\" : \"id\", (string) The transaction id (same as provided)\n" " \"hash\" : \"id\", (string) The transaction hash (differs from txid for witness transactions)\n" @@ -130,42 +137,56 @@ UniValue getrawtransaction(const JSONRPCRequest& request) + HelpExampleCli("getrawtransaction", "\"mytxid\"") + HelpExampleCli("getrawtransaction", "\"mytxid\" true") + HelpExampleRpc("getrawtransaction", "\"mytxid\", true") + + HelpExampleCli("getrawtransaction", "\"mytxid\" false \"myblockhash\"") + + HelpExampleCli("getrawtransaction", "\"mytxid\" true \"myblockhash\"") ); LOCK(cs_main); + bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); + CBlockIndex* blockindex = nullptr; // Accept either a bool (true) or a num (>=1) to indicate verbose output. bool fVerbose = false; if (!request.params[1].isNull()) { - if (request.params[1].isNum()) { - if (request.params[1].get_int() != 0) { - fVerbose = true; - } - } - else if(request.params[1].isBool()) { - if(request.params[1].isTrue()) { - fVerbose = true; - } - } - else { - throw JSONRPCError(RPC_TYPE_ERROR, "Invalid type provided. Verbose parameter must be a boolean."); + fVerbose = request.params[1].isNum() ? (request.params[1].get_int() != 0) : request.params[1].get_bool(); + } + + if (!request.params[2].isNull()) { + uint256 blockhash = ParseHashV(request.params[2], "parameter 3"); + BlockMap::iterator it = mapBlockIndex.find(blockhash); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block hash not found"); } + blockindex = it->second; + in_active_chain = chainActive.Contains(blockindex); } CTransactionRef tx; - uint256 hashBlock; - if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, std::string(fTxIndex ? "No such mempool or blockchain transaction" - : "No such mempool transaction. Use -txindex to enable blockchain transaction queries") + - ". Use gettransaction for wallet transactions."); + uint256 hash_block; + if (!GetTransaction(hash, tx, Params().GetConsensus(), hash_block, true, blockindex)) { + std::string errmsg; + if (blockindex) { + if (!(blockindex->nStatus & BLOCK_HAVE_DATA)) { + throw JSONRPCError(RPC_MISC_ERROR, "Block not available"); + } + errmsg = "No such transaction found in the provided block"; + } else { + errmsg = fTxIndex + ? "No such mempool or blockchain transaction" + : "No such mempool transaction. Use -txindex to enable blockchain transaction queries"; + } + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errmsg + ". Use gettransaction for wallet transactions."); + } - if (!fVerbose) + if (!fVerbose) { return EncodeHexTx(*tx, RPCSerializationFlags()); + } UniValue result(UniValue::VOBJ); - TxToJSON(*tx, hashBlock, result); + if (blockindex) result.push_back(Pair("in_active_chain", in_active_chain)); + TxToJSON(*tx, hash_block, result); return result; } @@ -328,7 +349,7 @@ UniValue createrawtransaction(const JSONRPCRequest& request) + HelpExampleRpc("createrawtransaction", "\"[{\\\"txid\\\":\\\"myid\\\",\\\"vout\\\":0}]\", \"{\\\"data\\\":\\\"00010203\\\"}\"") ); - RPCTypeCheck(request.params, {UniValue::VARR, UniValue::VOBJ, UniValue::VNUM}, true); + RPCTypeCheck(request.params, {UniValue::VARR, UniValue::VOBJ, UniValue::VNUM, UniValue::VBOOL}, true); if (request.params[0].isNull() || request.params[1].isNull()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, arguments 1 and 2 must be non-null"); @@ -420,13 +441,15 @@ UniValue createrawtransaction(const JSONRPCRequest& request) UniValue decoderawtransaction(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "decoderawtransaction \"hexstring\"\n" + "decoderawtransaction \"hexstring\" ( iswitness )\n" "\nReturn a JSON object representing the serialized, hex-encoded transaction.\n" "\nArguments:\n" "1. \"hexstring\" (string, required) The transaction hex string\n" + "2. iswitness (boolean, optional) Whether the transaction hex is a serialized witness transaction\n" + " If iswitness is not present, heuristic tests will be used in decoding\n" "\nResult:\n" "{\n" @@ -474,12 +497,16 @@ UniValue decoderawtransaction(const JSONRPCRequest& request) ); LOCK(cs_main); - RPCTypeCheck(request.params, {UniValue::VSTR}); + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL}); CMutableTransaction mtx; - if (!DecodeHexTx(mtx, request.params[0].get_str(), true)) + bool try_witness = request.params[1].isNull() ? true : request.params[1].get_bool(); + bool try_no_witness = request.params[1].isNull() ? true : !request.params[1].get_bool(); + + if (!DecodeHexTx(mtx, request.params[0].get_str(), try_no_witness, try_witness)) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); + } UniValue result(UniValue::VOBJ); TxToUniv(CTransaction(std::move(mtx)), uint256(), result, false); @@ -917,7 +944,9 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) ); ObserveSafeMode(); - LOCK(cs_main); + + std::promise<void> promise; + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL}); // parse hex string from parameter @@ -931,6 +960,8 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) if (!request.params[1].isNull() && request.params[1].get_bool()) nMaxRawTxFee = 0; + { // cs_main scope + LOCK(cs_main); CCoinsViewCache &view = *pcoinsTip; bool fHaveChain = false; for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { @@ -952,10 +983,28 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) } throw JSONRPCError(RPC_TRANSACTION_ERROR, state.GetRejectReason()); } + } else { + // If wallet is enabled, ensure that the wallet has been made aware + // of the new transaction prior to returning. This prevents a race + // where a user might call sendrawtransaction with a transaction + // to/from their wallet, immediately call some wallet RPC, and get + // a stale result because callbacks have not yet been processed. + CallFunctionInValidationInterfaceQueue([&promise] { + promise.set_value(); + }); } } else if (fHaveChain) { throw JSONRPCError(RPC_TRANSACTION_ALREADY_IN_CHAIN, "transaction already in block chain"); + } else { + // Make sure we don't block forever if re-sending + // a transaction already in mempool. + promise.set_value(); } + + } // cs_main + + promise.get_future().wait(); + if(!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); @@ -964,15 +1013,16 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) { pnode->PushInventory(inv); }); + return hashTx.GetHex(); } static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- - { "rawtransactions", "getrawtransaction", &getrawtransaction, {"txid","verbose"} }, + { "rawtransactions", "getrawtransaction", &getrawtransaction, {"txid","verbose","blockhash"} }, { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, - { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring"} }, + { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","allowhighfees"} }, { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, diff --git a/src/rpc/safemode.cpp b/src/rpc/safemode.cpp index 24770ad47f..9f3a9d30b8 100644 --- a/src/rpc/safemode.cpp +++ b/src/rpc/safemode.cpp @@ -1,8 +1,8 @@ -#include "safemode.h" +#include <rpc/safemode.h> -#include "rpc/protocol.h" -#include "util.h" -#include "warnings.h" +#include <rpc/protocol.h> +#include <util.h> +#include <warnings.h> void ObserveSafeMode() { diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index a73b697e01..e5b4f6ca77 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -1,20 +1,18 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/server.h" +#include <rpc/server.h> -#include "base58.h" -#include "fs.h" -#include "init.h" -#include "random.h" -#include "sync.h" -#include "ui_interface.h" -#include "util.h" -#include "utilstrencodings.h" - -#include <univalue.h> +#include <base58.h> +#include <fs.h> +#include <init.h> +#include <random.h> +#include <sync.h> +#include <ui_interface.h> +#include <util.h> +#include <utilstrencodings.h> #include <boost/bind.hpp> #include <boost/signals2/signal.hpp> @@ -160,8 +158,8 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& std::set<rpcfn_type> setDone; std::vector<std::pair<std::string, const CRPCCommand*> > vCommands; - for (std::map<std::string, const CRPCCommand*>::const_iterator mi = mapCommands.begin(); mi != mapCommands.end(); ++mi) - vCommands.push_back(make_pair(mi->second->category + mi->first, mi->second)); + for (const auto& entry : mapCommands) + vCommands.push_back(make_pair(entry.second->category + entry.first, entry.second)); sort(vCommands.begin(), vCommands.end()); JSONRPCRequest jreq(helpreq); @@ -389,11 +387,10 @@ bool IsDeprecatedRPCEnabled(const std::string& method) return find(enabled_methods.begin(), enabled_methods.end(), method) != enabled_methods.end(); } -static UniValue JSONRPCExecOne(const UniValue& req) +static UniValue JSONRPCExecOne(JSONRPCRequest jreq, const UniValue& req) { UniValue rpc_result(UniValue::VOBJ); - JSONRPCRequest jreq; try { jreq.parse(req); @@ -413,11 +410,11 @@ static UniValue JSONRPCExecOne(const UniValue& req) return rpc_result; } -std::string JSONRPCExecBatch(const UniValue& vReq) +std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq) { UniValue ret(UniValue::VARR); for (unsigned int reqIdx = 0; reqIdx < vReq.size(); reqIdx++) - ret.push_back(JSONRPCExecOne(vReq[reqIdx])); + ret.push_back(JSONRPCExecOne(jreq, vReq[reqIdx])); return ret.write() + "\n"; } diff --git a/src/rpc/server.h b/src/rpc/server.h index 31d6304271..075940cb90 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -1,14 +1,14 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_RPCSERVER_H #define BITCOIN_RPCSERVER_H -#include "amount.h" -#include "rpc/protocol.h" -#include "uint256.h" +#include <amount.h> +#include <rpc/protocol.h> +#include <uint256.h> #include <list> #include <map> @@ -191,7 +191,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri bool StartRPC(); void InterruptRPC(); void StopRPC(); -std::string JSONRPCExecBatch(const UniValue& vReq); +std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq); // Retrieves any serialization flags requested in command line argument int RPCSerializationFlags(); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 4edb2c6d9b..a94f6b2a66 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "scheduler.h" +#include <scheduler.h> -#include "random.h" -#include "reverselock.h" +#include <random.h> +#include <reverselock.h> #include <assert.h> #include <boost/bind.hpp> @@ -206,3 +206,8 @@ void SingleThreadedSchedulerClient::EmptyQueue() { should_continue = !m_callbacks_pending.empty(); } } + +size_t SingleThreadedSchedulerClient::CallbacksPending() { + LOCK(m_cs_callbacks_pending); + return m_callbacks_pending.size(); +} diff --git a/src/scheduler.h b/src/scheduler.h index db93bcb21e..a41838a295 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,7 +14,7 @@ #include <boost/thread.hpp> #include <map> -#include "sync.h" +#include <sync.h> // // Simple class for background tasks that should be run @@ -108,6 +108,8 @@ public: // Processes all remaining queue members on the calling thread, blocking until queue is empty // Must be called after the CScheduler has no remaining processing threads! void EmptyQueue(); + + size_t CallbacksPending(); }; #endif diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index 03128917fd..7d3587e2c2 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -1,14 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bitcoinconsensus.h" +#include <script/bitcoinconsensus.h> -#include "primitives/transaction.h" -#include "pubkey.h" -#include "script/interpreter.h" -#include "version.h" +#include <primitives/transaction.h> +#include <pubkey.h> +#include <script/interpreter.h> +#include <version.h> namespace { diff --git a/src/script/bitcoinconsensus.h b/src/script/bitcoinconsensus.h index 33bf80e5a7..bb94c17528 100644 --- a/src/script/bitcoinconsensus.h +++ b/src/script/bitcoinconsensus.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ #include <stdint.h> #if defined(BUILD_BITCOIN_INTERNAL) && defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #if defined(_WIN32) #if defined(DLL_EXPORT) #if defined(HAVE_FUNC_ATTRIBUTE_DLLEXPORT) diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 2f7b8e3a03..2cdff7ee57 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1,17 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "interpreter.h" +#include <script/interpreter.h> -#include "primitives/transaction.h" -#include "crypto/ripemd160.h" -#include "crypto/sha1.h" -#include "crypto/sha256.h" -#include "pubkey.h" -#include "script/script.h" -#include "uint256.h" +#include <crypto/ripemd160.h> +#include <crypto/sha1.h> +#include <crypto/sha256.h> +#include <pubkey.h> +#include <script/script.h> +#include <uint256.h> typedef std::vector<unsigned char> valtype; @@ -174,7 +173,13 @@ bool static IsLowDERSignature(const valtype &vchSig, ScriptError* serror) { if (!IsValidSignatureEncoding(vchSig)) { return set_error(serror, SCRIPT_ERR_SIG_DER); } + // https://bitcoin.stackexchange.com/a/12556: + // Also note that inside transaction signatures, an extra hashtype byte + // follows the actual signature data. std::vector<unsigned char> vchSigCopy(vchSig.begin(), vchSig.begin() + vchSig.size() - 1); + // If the S value is above the order of the curve divided by two, its + // complement modulo the order could have been used instead, which is + // one byte shorter when encoded correctly. if (!CPubKey::CheckLowS(vchSigCopy)) { return set_error(serror, SCRIPT_ERR_SIG_HIGH_S); } @@ -349,9 +354,6 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& { if (!(flags & SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY)) { // not enabled; treat as a NOP2 - if (flags & SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS) { - return set_error(serror, SCRIPT_ERR_DISCOURAGE_UPGRADABLE_NOPS); - } break; } @@ -391,9 +393,6 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& { if (!(flags & SCRIPT_VERIFY_CHECKSEQUENCEVERIFY)) { // not enabled; treat as a NOP3 - if (flags & SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS) { - return set_error(serror, SCRIPT_ERR_DISCOURAGE_UPGRADABLE_NOPS); - } break; } diff --git a/src/script/interpreter.h b/src/script/interpreter.h index 1cb9cc7899..e12329be76 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SCRIPT_INTERPRETER_H #define BITCOIN_SCRIPT_INTERPRETER_H -#include "script_error.h" -#include "primitives/transaction.h" +#include <script/script_error.h> +#include <primitives/transaction.h> #include <vector> #include <stdint.h> @@ -27,37 +27,40 @@ enum SIGHASH_ANYONECANPAY = 0x80, }; -/** Script verification flags */ +/** Script verification flags. + * + * All flags are intended to be soft forks: the set of acceptable scripts under + * flags (A | B) is a subset of the acceptable scripts under flag (A). + */ enum { SCRIPT_VERIFY_NONE = 0, - // Evaluate P2SH subscripts (softfork safe, BIP16). + // Evaluate P2SH subscripts (BIP16). SCRIPT_VERIFY_P2SH = (1U << 0), // Passing a non-strict-DER signature or one with undefined hashtype to a checksig operation causes script failure. // Evaluating a pubkey that is not (0x04 + 64 bytes) or (0x02 or 0x03 + 32 bytes) by checksig causes script failure. - // (softfork safe, but not used or intended as a consensus rule). + // (not used or intended as a consensus rule). SCRIPT_VERIFY_STRICTENC = (1U << 1), - // Passing a non-strict-DER signature to a checksig operation causes script failure (softfork safe, BIP62 rule 1) + // Passing a non-strict-DER signature to a checksig operation causes script failure (BIP62 rule 1) SCRIPT_VERIFY_DERSIG = (1U << 2), // Passing a non-strict-DER signature or one with S > order/2 to a checksig operation causes script failure - // (softfork safe, BIP62 rule 5). + // (BIP62 rule 5). SCRIPT_VERIFY_LOW_S = (1U << 3), - // verify dummy stack item consumed by CHECKMULTISIG is of zero-length (softfork safe, BIP62 rule 7). + // verify dummy stack item consumed by CHECKMULTISIG is of zero-length (BIP62 rule 7). SCRIPT_VERIFY_NULLDUMMY = (1U << 4), - // Using a non-push operator in the scriptSig causes script failure (softfork safe, BIP62 rule 2). + // Using a non-push operator in the scriptSig causes script failure (BIP62 rule 2). SCRIPT_VERIFY_SIGPUSHONLY = (1U << 5), // Require minimal encodings for all push operations (OP_0... OP_16, OP_1NEGATE where possible, direct // pushes up to 75 bytes, OP_PUSHDATA up to 255 bytes, OP_PUSHDATA2 for anything larger). Evaluating // any other push causes the script to fail (BIP62 rule 3). // In addition, whenever a stack element is interpreted as a number, it must be of minimal length (BIP62 rule 4). - // (softfork safe) SCRIPT_VERIFY_MINIMALDATA = (1U << 6), // Discourage use of NOPs reserved for upgrades (NOP1-10) @@ -68,12 +71,14 @@ enum // discouraged NOPs fails the script. This verification flag will never be // a mandatory flag applied to scripts in a block. NOPs that are not // executed, e.g. within an unexecuted IF ENDIF block, are *not* rejected. + // NOPs that have associated forks to give them new meaning (CLTV, CSV) + // are not subject to this rule. SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS = (1U << 7), // Require that only a single stack element remains after evaluation. This changes the success criterion from // "At least one stack element must remain, and when interpreted as a boolean, it must be true" to // "Exactly one stack element must remain, and when interpreted as a boolean, it must be true". - // (softfork safe, BIP62 rule 6) + // (BIP62 rule 6) // Note: CLEANSTACK should never be used without P2SH or WITNESS. SCRIPT_VERIFY_CLEANSTACK = (1U << 8), diff --git a/src/script/ismine.cpp b/src/script/ismine.cpp index 6b68f0679e..d0dd272550 100644 --- a/src/script/ismine.cpp +++ b/src/script/ismine.cpp @@ -1,15 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "ismine.h" +#include <script/ismine.h> -#include "key.h" -#include "keystore.h" -#include "script/script.h" -#include "script/standard.h" -#include "script/sign.h" +#include <key.h> +#include <keystore.h> +#include <script/script.h> +#include <script/sign.h> typedef std::vector<unsigned char> valtype; diff --git a/src/script/ismine.h b/src/script/ismine.h index 1aa5937b34..b54879cc15 100644 --- a/src/script/ismine.h +++ b/src/script/ismine.h @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SCRIPT_ISMINE_H #define BITCOIN_SCRIPT_ISMINE_H -#include "script/standard.h" +#include <script/standard.h> #include <stdint.h> diff --git a/src/script/script.cpp b/src/script/script.cpp index a10b619f7d..65e5405ebd 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "script.h" +#include <script/script.h> -#include "tinyformat.h" -#include "utilstrencodings.h" +#include <tinyformat.h> +#include <utilstrencodings.h> const char* GetOpName(opcodetype opcode) { diff --git a/src/script/script.h b/src/script/script.h index 2a92060543..bedf5f9be5 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -1,14 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SCRIPT_SCRIPT_H #define BITCOIN_SCRIPT_SCRIPT_H -#include "crypto/common.h" -#include "prevector.h" -#include "serialize.h" +#include <crypto/common.h> +#include <prevector.h> +#include <serialize.h> #include <assert.h> #include <climits> diff --git a/src/script/script_error.cpp b/src/script/script_error.cpp index 6c590f53e3..dbceb1f740 100644 --- a/src/script/script_error.cpp +++ b/src/script/script_error.cpp @@ -1,9 +1,9 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "script_error.h" +#include <script/script_error.h> const char* ScriptErrorString(const ScriptError serror) { diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 4cc7afa2f5..9638b12f99 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -1,17 +1,17 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "sigcache.h" +#include <script/sigcache.h> -#include "memusage.h" -#include "pubkey.h" -#include "random.h" -#include "uint256.h" -#include "util.h" +#include <memusage.h> +#include <pubkey.h> +#include <random.h> +#include <uint256.h> +#include <util.h> -#include "cuckoocache.h" +#include <cuckoocache.h> #include <boost/thread.hpp> namespace { diff --git a/src/script/sigcache.h b/src/script/sigcache.h index 5832b264b3..1309d57cc6 100644 --- a/src/script/sigcache.h +++ b/src/script/sigcache.h @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SCRIPT_SIGCACHE_H #define BITCOIN_SCRIPT_SIGCACHE_H -#include "script/interpreter.h" +#include <script/interpreter.h> #include <vector> diff --git a/src/script/sign.cpp b/src/script/sign.cpp index ac58b690a2..838e502a0a 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -1,16 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "script/sign.h" +#include <script/sign.h> -#include "key.h" -#include "keystore.h" -#include "policy/policy.h" -#include "primitives/transaction.h" -#include "script/standard.h" -#include "uint256.h" +#include <key.h> +#include <keystore.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <script/standard.h> +#include <uint256.h> typedef std::vector<unsigned char> valtype; @@ -422,3 +422,22 @@ bool DummySignatureCreator::CreateSig(std::vector<unsigned char>& vchSig, const vchSig[6 + 33 + 32] = SIGHASH_ALL; return true; } + +bool IsSolvable(const CKeyStore& store, const CScript& script) +{ + // This check is to make sure that the script we created can actually be solved for and signed by us + // if we were to have the private keys. This is just to make sure that the script is valid and that, + // if found in a transaction, we would still accept and relay that transaction. In particular, + // it will reject witness outputs that require signing with an uncompressed public key. + DummySignatureCreator creator(&store); + SignatureData sigs; + // Make sure that STANDARD_SCRIPT_VERIFY_FLAGS includes SCRIPT_VERIFY_WITNESS_PUBKEYTYPE, the most + // important property this function is designed to test for. + static_assert(STANDARD_SCRIPT_VERIFY_FLAGS & SCRIPT_VERIFY_WITNESS_PUBKEYTYPE, "IsSolvable requires standard script flags to include WITNESS_PUBKEYTYPE"); + if (ProduceSignature(creator, script, sigs)) { + // VerifyScript check is just defensive, and should never fail. + assert(VerifyScript(sigs.scriptSig, script, &sigs.scriptWitness, STANDARD_SCRIPT_VERIFY_FLAGS, creator.Checker())); + return true; + } + return false; +} diff --git a/src/script/sign.h b/src/script/sign.h index a0d8ee4ff9..97c0014cd0 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SCRIPT_SIGN_H #define BITCOIN_SCRIPT_SIGN_H -#include "script/interpreter.h" +#include <script/interpreter.h> class CKeyID; class CKeyStore; @@ -81,4 +81,10 @@ SignatureData CombineSignatures(const CScript& scriptPubKey, const BaseSignature SignatureData DataFromTransaction(const CMutableTransaction& tx, unsigned int nIn); void UpdateTransaction(CMutableTransaction& tx, unsigned int nIn, const SignatureData& data); +/* Check whether we know how to sign for an output like this, assuming we + * have all private keys. While this function does not need private keys, the passed + * keystore is used to look up public keys and redeemscripts by hash. + * Solvability is unrelated to whether we consider this output to be ours. */ +bool IsSolvable(const CKeyStore& store, const CScript& script); + #endif // BITCOIN_SCRIPT_SIGN_H diff --git a/src/script/standard.cpp b/src/script/standard.cpp index f57f1f61b4..cfb3c58588 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -1,14 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "script/standard.h" +#include <script/standard.h> -#include "pubkey.h" -#include "script/script.h" -#include "util.h" -#include "utilstrencodings.h" +#include <pubkey.h> +#include <script/script.h> +#include <util.h> +#include <utilstrencodings.h> typedef std::vector<unsigned char> valtype; @@ -348,19 +348,14 @@ CScript GetScriptForWitness(const CScript& redeemscript) std::vector<std::vector<unsigned char> > vSolutions; if (Solver(redeemscript, typ, vSolutions)) { if (typ == TX_PUBKEY) { - unsigned char h160[20]; - CHash160().Write(&vSolutions[0][0], vSolutions[0].size()).Finalize(h160); - ret << OP_0 << std::vector<unsigned char>(&h160[0], &h160[20]); - return ret; + return GetScriptForDestination(WitnessV0KeyHash(Hash160(vSolutions[0].begin(), vSolutions[0].end()))); } else if (typ == TX_PUBKEYHASH) { - ret << OP_0 << vSolutions[0]; - return ret; + return GetScriptForDestination(WitnessV0KeyHash(vSolutions[0])); } } uint256 hash; CSHA256().Write(&redeemscript[0], redeemscript.size()).Finalize(hash.begin()); - ret << OP_0 << ToByteVector(hash); - return ret; + return GetScriptForDestination(WitnessV0ScriptHash(hash)); } bool IsValidDestination(const CTxDestination& dest) { diff --git a/src/script/standard.h b/src/script/standard.h index fa07ea88c1..3b2838a5bb 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SCRIPT_STANDARD_H #define BITCOIN_SCRIPT_STANDARD_H -#include "script/interpreter.h" -#include "uint256.h" +#include <script/interpreter.h> +#include <uint256.h> #include <boost/variant.hpp> @@ -73,8 +73,19 @@ public: friend bool operator<(const CNoDestination &a, const CNoDestination &b) { return true; } }; -struct WitnessV0ScriptHash : public uint256 {}; -struct WitnessV0KeyHash : public uint160 {}; +struct WitnessV0ScriptHash : public uint256 +{ + WitnessV0ScriptHash() : uint256() {} + explicit WitnessV0ScriptHash(const uint256& hash) : uint256(hash) {} + using uint256::uint256; +}; + +struct WitnessV0KeyHash : public uint160 +{ + WitnessV0KeyHash() : uint160() {} + explicit WitnessV0KeyHash(const uint160& hash) : uint160(hash) {} + using uint160::uint160; +}; //! CTxDestination subtype to encode any future Witness version struct WitnessUnknown @@ -144,6 +155,10 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) * addressRet is populated with a single value and nRequiredRet is set to 1. * Returns true if successful. Currently does not extract address from * pay-to-witness scripts. + * + * Note: this function confuses destinations (a subset of CScripts that are + * encodable as an address) with key identifiers (of keys involved in a + * CScript), and its use should be phased out. */ bool ExtractDestinations(const CScript& scriptPubKey, txnouttype& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet); @@ -164,6 +179,9 @@ CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys); * Generate a pay-to-witness script for the given redeem script. If the redeem * script is P2PK or P2PKH, this returns a P2WPKH script, otherwise it returns a * P2WSH script. + * + * TODO: replace calls to GetScriptForWitness with GetScriptForDestination using + * the various witness-specific CTxDestination subtypes. */ CScript GetScriptForWitness(const CScript& redeemscript); diff --git a/src/serialize.h b/src/serialize.h index eeb05fa76c..dcc8d8691e 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SERIALIZE_H #define BITCOIN_SERIALIZE_H -#include "compat/endian.h" +#include <compat/endian.h> #include <algorithm> #include <assert.h> @@ -21,7 +21,7 @@ #include <utility> #include <vector> -#include "prevector.h" +#include <prevector.h> static const unsigned int MAX_SIZE = 0x02000000; @@ -732,8 +732,8 @@ template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m) { WriteCompactSize(os, m.size()); - for (typename std::map<K, T, Pred, A>::const_iterator mi = m.begin(); mi != m.end(); ++mi) - Serialize(os, (*mi)); + for (const auto& entry : m) + Serialize(os, entry); } template<typename Stream, typename K, typename T, typename Pred, typename A> diff --git a/src/streams.h b/src/streams.h index 9a3badea57..98e25178e1 100644 --- a/src/streams.h +++ b/src/streams.h @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_STREAMS_H #define BITCOIN_STREAMS_H -#include "support/allocators/zeroafterfree.h" -#include "serialize.h" +#include <support/allocators/zeroafterfree.h> +#include <serialize.h> #include <algorithm> #include <assert.h> @@ -241,8 +241,8 @@ public: const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } reference operator[](size_type pos) { return vch[pos + nReadPos]; } void clear() { vch.clear(); nReadPos = 0; } - iterator insert(iterator it, const char& x=char()) { return vch.insert(it, x); } - void insert(iterator it, size_type n, const char& x) { vch.insert(it, n, x); } + iterator insert(iterator it, const char x=char()) { return vch.insert(it, x); } + void insert(iterator it, size_type n, const char x) { vch.insert(it, n, x); } value_type* data() { return vch.data() + nReadPos; } const value_type* data() const { return vch.data() + nReadPos; } @@ -345,18 +345,16 @@ public: // Read from the beginning of the buffer unsigned int nReadPosNext = nReadPos + nSize; - if (nReadPosNext >= vch.size()) + if (nReadPosNext > vch.size()) { + throw std::ios_base::failure("CDataStream::read(): end of data"); + } + memcpy(pch, &vch[nReadPos], nSize); + if (nReadPosNext == vch.size()) { - if (nReadPosNext > vch.size()) - { - throw std::ios_base::failure("CDataStream::read(): end of data"); - } - memcpy(pch, &vch[nReadPos], nSize); nReadPos = 0; vch.clear(); return; } - memcpy(pch, &vch[nReadPos], nSize); nReadPos = nReadPosNext; } diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 39347c73bb..0249afa499 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SUPPORT_ALLOCATORS_SECURE_H #define BITCOIN_SUPPORT_ALLOCATORS_SECURE_H -#include "support/lockedpool.h" -#include "support/cleanse.h" +#include <support/lockedpool.h> +#include <support/cleanse.h> #include <string> diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index 618874ceee..78ff1ad024 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H #define BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H -#include "support/cleanse.h" +#include <support/cleanse.h> #include <memory> #include <vector> diff --git a/src/support/cleanse.cpp b/src/support/cleanse.cpp index 95899c9f02..9bf7eda785 100644 --- a/src/support/cleanse.cpp +++ b/src/support/cleanse.cpp @@ -1,12 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "cleanse.h" +#include <support/cleanse.h> #include <cstring> +#if defined(_MSC_VER) +#include <Windows.h> // For SecureZeroMemory. +#endif + /* Compilers have a bad habit of removing "superfluous" memset calls that * are trying to zero memory. For example, when memset()ing a buffer and * then free()ing it, the compiler might decide that the memset is @@ -32,7 +36,7 @@ void memory_cleanse(void *ptr, size_t len) might try to eliminate "superfluous" memsets. If there's an easy way to detect memset_s, it would be better to use that. */ #if defined(_MSC_VER) - __asm; + SecureZeroMemory(ptr, len); #else __asm__ __volatile__("" : : "r"(ptr) : "memory"); #endif diff --git a/src/support/cleanse.h b/src/support/cleanse.h index f020216c73..69c27df519 100644 --- a/src/support/cleanse.h +++ b/src/support/cleanse.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/events.h b/src/support/events.h index cc6d29aecd..7ad51f319c 100644 --- a/src/support/events.h +++ b/src/support/events.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index 2ead72185f..d92ab02d6b 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "support/lockedpool.h" -#include "support/cleanse.h" +#include <support/lockedpool.h> +#include <support/cleanse.h> #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #ifdef WIN32 diff --git a/src/support/lockedpool.h b/src/support/lockedpool.h index 834f0371e2..fc85e6c73c 100644 --- a/src/support/lockedpool.h +++ b/src/support/lockedpool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/sync.cpp b/src/sync.cpp index 87024ccdf2..ae6e721466 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -1,17 +1,19 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "sync.h" +#include <sync.h> -#include "util.h" -#include "utilstrencodings.h" +#include <set> +#include <util.h> +#include <utilstrencodings.h> #include <stdio.h> -#include <boost/thread.hpp> - #ifdef DEBUG_LOCKCONTENTION +#if !defined(HAVE_THREAD_LOCAL) +static_assert(false, "thread_local is not supported"); +#endif void PrintLockContention(const char* pszName, const char* pszFile, int nLine) { LogPrintf("LOCKCONTENTION: %s\n", pszName); @@ -45,8 +47,8 @@ struct CLockLocation { return mutexName + " " + sourceFile + ":" + itostr(sourceLine) + (fTry ? " (TRY)" : ""); } - bool fTry; private: + bool fTry; std::string mutexName; std::string sourceFile; int sourceLine; @@ -67,10 +69,10 @@ struct LockData { LockOrders lockorders; InvLockOrders invlockorders; - boost::mutex dd_mutex; + std::mutex dd_mutex; } static lockdata; -boost::thread_specific_ptr<LockStack> lockstack; +static thread_local std::unique_ptr<LockStack> lockstack; static void potential_deadlock_detected(const std::pair<void*, void*>& mismatch, const LockStack& s1, const LockStack& s2) { @@ -100,12 +102,12 @@ static void potential_deadlock_detected(const std::pair<void*, void*>& mismatch, static void push_lock(void* c, const CLockLocation& locklocation) { - if (lockstack.get() == nullptr) + if (!lockstack) lockstack.reset(new LockStack); - boost::unique_lock<boost::mutex> lock(lockdata.dd_mutex); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); - (*lockstack).push_back(std::make_pair(c, locklocation)); + lockstack->push_back(std::make_pair(c, locklocation)); for (const std::pair<void*, CLockLocation> & i : (*lockstack)) { if (i.first == c) @@ -155,13 +157,23 @@ void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, abort(); } +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) +{ + for (const std::pair<void*, CLockLocation>& i : *lockstack) { + if (i.first == cs) { + fprintf(stderr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld().c_str()); + abort(); + } + } +} + void DeleteLock(void* cs) { if (!lockdata.available) { // We're already shutting down. return; } - boost::unique_lock<boost::mutex> lock(lockdata.dd_mutex); + std::lock_guard<std::mutex> lock(lockdata.dd_mutex); std::pair<void*, void*> item = std::make_pair(cs, nullptr); LockOrders::iterator it = lockdata.lockorders.lower_bound(item); while (it != lockdata.lockorders.end() && it->first.first == cs) { diff --git a/src/sync.h b/src/sync.h index 0871c5fb4d..3c451af376 100644 --- a/src/sync.h +++ b/src/sync.h @@ -1,16 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SYNC_H #define BITCOIN_SYNC_H -#include "threadsafety.h" +#include <threadsafety.h> -#include <boost/thread/condition_variable.hpp> -#include <boost/thread/mutex.hpp> -#include <boost/thread/recursive_mutex.hpp> +#include <condition_variable> +#include <thread> +#include <mutex> //////////////////////////////////////////////// @@ -21,17 +21,17 @@ /* CCriticalSection mutex; - boost::recursive_mutex mutex; + std::recursive_mutex mutex; LOCK(mutex); - boost::unique_lock<boost::recursive_mutex> criticalblock(mutex); + std::unique_lock<std::recursive_mutex> criticalblock(mutex); LOCK2(mutex1, mutex2); - boost::unique_lock<boost::recursive_mutex> criticalblock1(mutex1); - boost::unique_lock<boost::recursive_mutex> criticalblock2(mutex2); + std::unique_lock<std::recursive_mutex> criticalblock1(mutex1); + std::unique_lock<std::recursive_mutex> criticalblock2(mutex2); TRY_LOCK(mutex, name); - boost::unique_lock<boost::recursive_mutex> name(mutex, boost::try_to_lock_t); + std::unique_lock<std::recursive_mutex> name(mutex, std::try_to_lock_t); ENTER_CRITICAL_SECTION(mutex); // no RAII mutex.lock(); @@ -75,20 +75,23 @@ void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs void LeaveCritical(); std::string LocksHeld(); void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); void DeleteLock(void* cs); #else void static inline EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) {} void static inline LeaveCritical() {} void static inline AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} +void static inline AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} void static inline DeleteLock(void* cs) {} #endif #define AssertLockHeld(cs) AssertLockHeldInternal(#cs, __FILE__, __LINE__, &cs) +#define AssertLockNotHeld(cs) AssertLockNotHeldInternal(#cs, __FILE__, __LINE__, &cs) /** - * Wrapped boost mutex: supports recursive locking, but no waiting + * Wrapped mutex: supports recursive locking, but no waiting * TODO: We should move away from using the recursive lock by default. */ -class CCriticalSection : public AnnotatedMixin<boost::recursive_mutex> +class CCriticalSection : public AnnotatedMixin<std::recursive_mutex> { public: ~CCriticalSection() { @@ -96,22 +99,24 @@ public: } }; -/** Wrapped boost mutex: supports waiting but not recursive locking */ -typedef AnnotatedMixin<boost::mutex> CWaitableCriticalSection; +/** Wrapped mutex: supports waiting but not recursive locking */ +typedef AnnotatedMixin<std::mutex> CWaitableCriticalSection; -/** Just a typedef for boost::condition_variable, can be wrapped later if desired */ -typedef boost::condition_variable CConditionVariable; +/** Just a typedef for std::condition_variable, can be wrapped later if desired */ +typedef std::condition_variable CConditionVariable; + +/** Just a typedef for std::unique_lock, can be wrapped later if desired */ +typedef std::unique_lock<std::mutex> WaitableLock; #ifdef DEBUG_LOCKCONTENTION void PrintLockContention(const char* pszName, const char* pszFile, int nLine); #endif -/** Wrapper around boost::unique_lock<Mutex> */ -template <typename Mutex> -class SCOPED_LOCKABLE CMutexLock +/** Wrapper around std::unique_lock<CCriticalSection> */ +class SCOPED_LOCKABLE CCriticalBlock { private: - boost::unique_lock<Mutex> lock; + std::unique_lock<CCriticalSection> lock; void Enter(const char* pszName, const char* pszFile, int nLine) { @@ -136,7 +141,7 @@ private: } public: - CMutexLock(Mutex& mutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : lock(mutexIn, boost::defer_lock) + CCriticalBlock(CCriticalSection& mutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : lock(mutexIn, std::defer_lock) { if (fTry) TryEnter(pszName, pszFile, nLine); @@ -144,18 +149,18 @@ public: Enter(pszName, pszFile, nLine); } - CMutexLock(Mutex* pmutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) + CCriticalBlock(CCriticalSection* pmutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) { if (!pmutexIn) return; - lock = boost::unique_lock<Mutex>(*pmutexIn, boost::defer_lock); + lock = std::unique_lock<CCriticalSection>(*pmutexIn, std::defer_lock); if (fTry) TryEnter(pszName, pszFile, nLine); else Enter(pszName, pszFile, nLine); } - ~CMutexLock() UNLOCK_FUNCTION() + ~CCriticalBlock() UNLOCK_FUNCTION() { if (lock.owns_lock()) LeaveCritical(); @@ -167,8 +172,6 @@ public: } }; -typedef CMutexLock<CCriticalSection> CCriticalBlock; - #define PASTE(x, y) x ## y #define PASTE2(x, y) PASTE(x, y) @@ -191,8 +194,8 @@ typedef CMutexLock<CCriticalSection> CCriticalBlock; class CSemaphore { private: - boost::condition_variable condition; - boost::mutex mutex; + std::condition_variable condition; + std::mutex mutex; int value; public: @@ -200,16 +203,14 @@ public: void wait() { - boost::unique_lock<boost::mutex> lock(mutex); - while (value < 1) { - condition.wait(lock); - } + std::unique_lock<std::mutex> lock(mutex); + condition.wait(lock, [&]() { return value >= 1; }); value--; } bool try_wait() { - boost::unique_lock<boost::mutex> lock(mutex); + std::lock_guard<std::mutex> lock(mutex); if (value < 1) return false; value--; @@ -219,7 +220,7 @@ public: void post() { { - boost::unique_lock<boost::mutex> lock(mutex); + std::lock_guard<std::mutex> lock(mutex); value++; } condition.notify_one(); diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index b88ad5ed1b..abc31e6181 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -1,20 +1,20 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // Unit tests for denial-of-service detection/prevention code -#include "chainparams.h" -#include "keystore.h" -#include "net.h" -#include "net_processing.h" -#include "pow.h" -#include "script/sign.h" -#include "serialize.h" -#include "util.h" -#include "validation.h" +#include <chainparams.h> +#include <keystore.h> +#include <net.h> +#include <net_processing.h> +#include <pow.h> +#include <script/sign.h> +#include <serialize.h> +#include <util.h> +#include <validation.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <stdint.h> @@ -40,8 +40,141 @@ CService ip(uint32_t i) static NodeId id = 0; +void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds); + BOOST_FIXTURE_TEST_SUITE(DoS_tests, TestingSetup) +// Test eviction of an outbound peer whose chain never advances +// Mock a node connection, and use mocktime to simulate a peer +// which never sends any headers messages. PeerLogic should +// decide to evict that outbound peer, after the appropriate timeouts. +// Note that we protect 4 outbound nodes from being subject to +// this logic; this test takes advantage of that protection only +// being applied to nodes which send headers with sufficient +// work. +BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) +{ + std::atomic<bool> interruptDummy(false); + + // Mock an outbound peer + CAddress addr1(ip(0xa0b0c001), NODE_NONE); + CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", /*fInboundIn=*/ false); + dummyNode1.SetSendVersion(PROTOCOL_VERSION); + + peerLogic->InitializeNode(&dummyNode1); + dummyNode1.nVersion = 1; + dummyNode1.fSuccessfullyConnected = true; + + // This test requires that we have a chain with non-zero work. + LOCK(cs_main); + BOOST_CHECK(chainActive.Tip() != nullptr); + BOOST_CHECK(chainActive.Tip()->nChainWork > 0); + + // Test starts here + LOCK(dummyNode1.cs_sendProcessing); + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in getheaders + LOCK(dummyNode1.cs_vSend); + BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); + dummyNode1.vSendMsg.clear(); + + int64_t nStartTime = GetTime(); + // Wait 21 minutes + SetMockTime(nStartTime+21*60); + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in getheaders + BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); + // Wait 3 more minutes + SetMockTime(nStartTime+24*60); + peerLogic->SendMessages(&dummyNode1, interruptDummy); // should result in disconnect + BOOST_CHECK(dummyNode1.fDisconnect == true); + SetMockTime(0); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); +} + +void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerLogicValidation &peerLogic) +{ + CAddress addr(ip(GetRandInt(0xffffffff)), NODE_NONE); + vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", /*fInboundIn=*/ false)); + CNode &node = *vNodes.back(); + node.SetSendVersion(PROTOCOL_VERSION); + + peerLogic.InitializeNode(&node); + node.nVersion = 1; + node.fSuccessfullyConnected = true; + + CConnmanTest::AddNode(node); +} + +BOOST_AUTO_TEST_CASE(stale_tip_peer_management) +{ + const Consensus::Params& consensusParams = Params().GetConsensus(); + constexpr int nMaxOutbound = 8; + CConnman::Options options; + options.nMaxConnections = 125; + options.nMaxOutbound = nMaxOutbound; + options.nMaxFeeler = 1; + + connman->Init(options); + std::vector<CNode *> vNodes; + + // Mock some outbound peers + for (int i=0; i<nMaxOutbound; ++i) { + AddRandomOutboundPeer(vNodes, *peerLogic); + } + + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + + // No nodes should be marked for disconnection while we have no extra peers + for (const CNode *node : vNodes) { + BOOST_CHECK(node->fDisconnect == false); + } + + SetMockTime(GetTime() + 3*consensusParams.nPowTargetSpacing + 1); + + // Now tip should definitely be stale, and we should look for an extra + // outbound peer + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + BOOST_CHECK(connman->GetTryNewOutboundPeer()); + + // Still no peers should be marked for disconnection + for (const CNode *node : vNodes) { + BOOST_CHECK(node->fDisconnect == false); + } + + // If we add one more peer, something should get marked for eviction + // on the next check (since we're mocking the time to be in the future, the + // required time connected check should be satisfied). + AddRandomOutboundPeer(vNodes, *peerLogic); + + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + for (int i=0; i<nMaxOutbound; ++i) { + BOOST_CHECK(vNodes[i]->fDisconnect == false); + } + // Last added node should get marked for eviction + BOOST_CHECK(vNodes.back()->fDisconnect == true); + + vNodes.back()->fDisconnect = false; + + // Update the last announced block time for the last + // peer, and check that the next newest node gets evicted. + UpdateLastBlockAnnounceTime(vNodes.back()->GetId(), GetTime()); + + peerLogic->CheckForStaleTipAndEvictPeers(consensusParams); + for (int i=0; i<nMaxOutbound-1; ++i) { + BOOST_CHECK(vNodes[i]->fDisconnect == false); + } + BOOST_CHECK(vNodes[nMaxOutbound-1]->fDisconnect == true); + BOOST_CHECK(vNodes.back()->fDisconnect == false); + + bool dummy; + for (const CNode *node : vNodes) { + peerLogic->FinalizeNode(node->GetId(), dummy); + } + + CConnmanTest::ClearNodes(); +} + BOOST_AUTO_TEST_CASE(DoS_banning) { std::atomic<bool> interruptDummy(false); @@ -53,7 +186,11 @@ BOOST_AUTO_TEST_CASE(DoS_banning) peerLogic->InitializeNode(&dummyNode1); dummyNode1.nVersion = 1; dummyNode1.fSuccessfullyConnected = true; - Misbehaving(dummyNode1.GetId(), 100); // Should get banned + { + LOCK(cs_main); + Misbehaving(dummyNode1.GetId(), 100); // Should get banned + } + LOCK(dummyNode1.cs_sendProcessing); peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(connman->IsBanned(addr1)); BOOST_CHECK(!connman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned @@ -64,13 +201,24 @@ BOOST_AUTO_TEST_CASE(DoS_banning) peerLogic->InitializeNode(&dummyNode2); dummyNode2.nVersion = 1; dummyNode2.fSuccessfullyConnected = true; - Misbehaving(dummyNode2.GetId(), 50); + { + LOCK(cs_main); + Misbehaving(dummyNode2.GetId(), 50); + } + LOCK(dummyNode2.cs_sendProcessing); peerLogic->SendMessages(&dummyNode2, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr2)); // 2 not banned yet... BOOST_CHECK(connman->IsBanned(addr1)); // ... but 1 still should be - Misbehaving(dummyNode2.GetId(), 50); + { + LOCK(cs_main); + Misbehaving(dummyNode2.GetId(), 50); + } peerLogic->SendMessages(&dummyNode2, interruptDummy); BOOST_CHECK(connman->IsBanned(addr2)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); + peerLogic->FinalizeNode(dummyNode2.GetId(), dummy); } BOOST_AUTO_TEST_CASE(DoS_banscore) @@ -85,16 +233,29 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) peerLogic->InitializeNode(&dummyNode1); dummyNode1.nVersion = 1; dummyNode1.fSuccessfullyConnected = true; - Misbehaving(dummyNode1.GetId(), 100); + { + LOCK(cs_main); + Misbehaving(dummyNode1.GetId(), 100); + } + LOCK(dummyNode1.cs_sendProcessing); peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr1)); - Misbehaving(dummyNode1.GetId(), 10); + { + LOCK(cs_main); + Misbehaving(dummyNode1.GetId(), 10); + } peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(!connman->IsBanned(addr1)); - Misbehaving(dummyNode1.GetId(), 1); + { + LOCK(cs_main); + Misbehaving(dummyNode1.GetId(), 1); + } peerLogic->SendMessages(&dummyNode1, interruptDummy); BOOST_CHECK(connman->IsBanned(addr1)); gArgs.ForceSetArg("-banscore", std::to_string(DEFAULT_BANSCORE_THRESHOLD)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode1.GetId(), dummy); } BOOST_AUTO_TEST_CASE(DoS_bantime) @@ -112,7 +273,11 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) dummyNode.nVersion = 1; dummyNode.fSuccessfullyConnected = true; - Misbehaving(dummyNode.GetId(), 100); + { + LOCK(cs_main); + Misbehaving(dummyNode.GetId(), 100); + } + LOCK(dummyNode.cs_sendProcessing); peerLogic->SendMessages(&dummyNode, interruptDummy); BOOST_CHECK(connman->IsBanned(addr)); @@ -121,11 +286,15 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime+60*60*24+1); BOOST_CHECK(!connman->IsBanned(addr)); + + bool dummy; + peerLogic->FinalizeNode(dummyNode.GetId(), dummy); } CTransactionRef RandomOrphan() { std::map<uint256, COrphanTx>::iterator it; + LOCK(cs_main); it = mapOrphanTransactions.lower_bound(InsecureRand256()); if (it == mapOrphanTransactions.end()) it = mapOrphanTransactions.begin(); @@ -195,6 +364,7 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) BOOST_CHECK(!AddOrphanTx(MakeTransactionRef(tx), i)); } + LOCK(cs_main); // Test EraseOrphansFor: for (NodeId i = 0; i < 3; i++) { diff --git a/src/test/README.md b/src/test/README.md index dbaa9c27f3..01da32109b 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -52,4 +52,4 @@ examine `uint256_tests.cpp`. For further reading, I found the following website to be helpful in explaining how the boost unit test framework works: -[http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/](http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/). +[http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/](http://archive.is/dRBGf). diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 7be29c6d6b..b338d6d366 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addrman.h" -#include "test/test_bitcoin.h" +#include <addrman.h> +#include <test/test_bitcoin.h> #include <string> #include <boost/test/unit_test.hpp> -#include "hash.h" -#include "netbase.h" -#include "random.h" +#include <hash.h> +#include <netbase.h> +#include <random.h> class CAddrManTest : public CAddrMan { diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index 4a533b5bf2..c177f0bf00 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "util.h" +#include <util.h> -#include "support/allocators/secure.h" -#include "test/test_bitcoin.h" +#include <support/allocators/secure.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index 952cf901f0..854146102e 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "amount.h" -#include "policy/feerate.h" -#include "test/test_bitcoin.h" +#include <amount.h> +#include <policy/feerate.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index 2c98fbcfd6..21a1153ad0 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,11 +8,11 @@ #include <iomanip> #include <limits> #include <cmath> -#include "uint256.h" -#include "arith_uint256.h" +#include <uint256.h> +#include <arith_uint256.h> #include <string> -#include "version.h" -#include "test/test_bitcoin.h" +#include <version.h> +#include <test/test_bitcoin.h> BOOST_FIXTURE_TEST_SUITE(arith_uint256_tests, BasicTestingSetup) diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index 6422b3a88f..b9ac62a437 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2015 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index 6bc6dd5187..a2d4f82695 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -1,19 +1,19 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" +#include <base58.h> -#include "data/base58_encode_decode.json.h" -#include "data/base58_keys_invalid.json.h" -#include "data/base58_keys_valid.json.h" +#include <test/data/base58_encode_decode.json.h> +#include <test/data/base58_keys_invalid.json.h> +#include <test/data/base58_keys_valid.json.h> -#include "key.h" -#include "script/script.h" -#include "test/test_bitcoin.h" -#include "uint256.h" -#include "util.h" -#include "utilstrencodings.h" +#include <key.h> +#include <script/script.h> +#include <test/test_bitcoin.h> +#include <uint256.h> +#include <util.h> +#include <utilstrencodings.h> #include <univalue.h> diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index ccad94d946..b31f73f3b5 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/bech32_tests.cpp b/src/test/bech32_tests.cpp index ce4cddd64b..495290c8d9 100644 --- a/src/test/bech32_tests.cpp +++ b/src/test/bech32_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bech32.h" -#include "test/test_bitcoin.h" +#include <bech32.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index e123c26ad0..438ddc177d 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2013-2015 The Bitcoin Core developers +// Copyright (c) 2013-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> -#include "base58.h" -#include "key.h" -#include "uint256.h" -#include "util.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <base58.h> +#include <key.h> +#include <uint256.h> +#include <util.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <string> #include <vector> diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp new file mode 100644 index 0000000000..55fdd2c071 --- /dev/null +++ b/src/test/blockchain_tests.cpp @@ -0,0 +1,126 @@ +#include <boost/test/unit_test.hpp> + +#include "stdlib.h" + +#include "rpc/blockchain.cpp" +#include "test/test_bitcoin.h" + +/* Equality between doubles is imprecise. Comparison should be done + * with a small threshold of tolerance, rather than exact equality. + */ +bool DoubleEquals(double a, double b, double epsilon) +{ + return std::abs(a - b) < epsilon; +} + +CBlockIndex* CreateBlockIndexWithNbits(uint32_t nbits) +{ + CBlockIndex* block_index = new CBlockIndex(); + block_index->nHeight = 46367; + block_index->nTime = 1269211443; + block_index->nBits = nbits; + return block_index; +} + +CChain CreateChainWithNbits(uint32_t nbits) +{ + CBlockIndex* block_index = CreateBlockIndexWithNbits(nbits); + CChain chain; + chain.SetTip(block_index); + return chain; +} + +void RejectDifficultyMismatch(double difficulty, double expected_difficulty) { + BOOST_CHECK_MESSAGE( + DoubleEquals(difficulty, expected_difficulty, 0.00001), + "Difficulty was " + std::to_string(difficulty) + + " but was expected to be " + std::to_string(expected_difficulty)); +} + +/* Given a BlockIndex with the provided nbits, + * verify that the expected difficulty results. + */ +void TestDifficulty(uint32_t nbits, double expected_difficulty) +{ + CBlockIndex* block_index = CreateBlockIndexWithNbits(nbits); + /* Since we are passing in block index explicitly, + * there is no need to set up anything within the chain itself. + */ + CChain chain; + + double difficulty = GetDifficulty(chain, block_index); + delete block_index; + + RejectDifficultyMismatch(difficulty, expected_difficulty); +} + +BOOST_FIXTURE_TEST_SUITE(blockchain_difficulty_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(get_difficulty_for_very_low_target) +{ + TestDifficulty(0x1f111111, 0.000001); +} + +BOOST_AUTO_TEST_CASE(get_difficulty_for_low_target) +{ + TestDifficulty(0x1ef88f6f, 0.000016); +} + +BOOST_AUTO_TEST_CASE(get_difficulty_for_mid_target) +{ + TestDifficulty(0x1df88f6f, 0.004023); +} + +BOOST_AUTO_TEST_CASE(get_difficulty_for_high_target) +{ + TestDifficulty(0x1cf88f6f, 1.029916); +} + +BOOST_AUTO_TEST_CASE(get_difficulty_for_very_high_target) +{ + TestDifficulty(0x12345678, 5913134931067755359633408.0); +} + +// Verify that difficulty is 1.0 for an empty chain. +BOOST_AUTO_TEST_CASE(get_difficulty_for_null_tip) +{ + CChain chain; + double difficulty = GetDifficulty(chain, nullptr); + RejectDifficultyMismatch(difficulty, 1.0); +} + +/* Verify that if difficulty is based upon the block index + * in the chain, if no block index is explicitly specified. + */ +BOOST_AUTO_TEST_CASE(get_difficulty_for_null_block_index) +{ + CChain chain = CreateChainWithNbits(0x1df88f6f); + + double difficulty = GetDifficulty(chain, nullptr); + delete chain.Tip(); + + double expected_difficulty = 0.004023; + + RejectDifficultyMismatch(difficulty, expected_difficulty); +} + +/* Verify that difficulty is based upon the explicitly specified + * block index rather than being taken from the provided chain, + * when both are present. + */ +BOOST_AUTO_TEST_CASE(get_difficulty_for_block_index_overrides_tip) +{ + CChain chain = CreateChainWithNbits(0x1df88f6f); + /* This block index's nbits should be used + * instead of the chain's when calculating difficulty. + */ + CBlockIndex* override_block_index = CreateBlockIndexWithNbits(0x12345678); + + double difficulty = GetDifficulty(chain, override_block_index); + delete chain.Tip(); + delete override_block_index; + + RejectDifficultyMismatch(difficulty, 5913134931067755359633408.0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index f2d5b385d0..32330e0548 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "blockencodings.h" -#include "consensus/merkle.h" -#include "chainparams.h" -#include "random.h" +#include <blockencodings.h> +#include <consensus/merkle.h> +#include <chainparams.h> +#include <random.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> @@ -62,6 +62,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) CBlock block(BuildBlockTestCase()); pool.addUnchecked(block.vtx[2]->GetHash(), entry.FromTx(*block.vtx[2])); + LOCK(pool.cs); BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); // Do a simple ShortTxIDs RT @@ -161,6 +162,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) CBlock block(BuildBlockTestCase()); pool.addUnchecked(block.vtx[2]->GetHash(), entry.FromTx(*block.vtx[2])); + LOCK(pool.cs); BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -227,6 +229,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) CBlock block(BuildBlockTestCase()); pool.addUnchecked(block.vtx[1]->GetHash(), entry.FromTx(*block.vtx[1])); + LOCK(pool.cs); BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index eac2c102a6..af5533b109 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bloom.h" - -#include "base58.h" -#include "clientversion.h" -#include "key.h" -#include "merkleblock.h" -#include "primitives/block.h" -#include "random.h" -#include "serialize.h" -#include "streams.h" -#include "uint256.h" -#include "util.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <bloom.h> + +#include <base58.h> +#include <clientversion.h> +#include <key.h> +#include <merkleblock.h> +#include <primitives/block.h> +#include <random.h> +#include <serialize.h> +#include <streams.h> +#include <uint256.h> +#include <util.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/bswap_tests.cpp b/src/test/bswap_tests.cpp index 7b3134d327..f22f513e87 100644 --- a/src/test/bswap_tests.cpp +++ b/src/test/bswap_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "compat/byteswap.h" -#include "test/test_bitcoin.h" +#include <compat/byteswap.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 6ae0bcadd0..42f9dd0600 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -2,12 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "util.h" -#include "utiltime.h" -#include "validation.h" +#include <util.h> +#include <utiltime.h> +#include <validation.h> -#include "test/test_bitcoin.h" -#include "checkqueue.h" +#include <test/test_bitcoin.h> +#include <checkqueue.h> #include <boost/test/unit_test.hpp> #include <boost/thread.hpp> #include <atomic> @@ -18,13 +18,13 @@ #include <unordered_set> #include <memory> -#include "random.h" +#include <random.h> // BasicTestingSetup not sufficient because nScriptCheckThreads is not set // otherwise. BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, TestingSetup) -static const int QUEUE_BATCH_SIZE = 128; +static const unsigned int QUEUE_BATCH_SIZE = 128; struct FakeCheck { bool operator()() @@ -38,7 +38,7 @@ struct FakeCheckCheckCompletion { static std::atomic<size_t> n_calls; bool operator()() { - ++n_calls; + n_calls.fetch_add(1, std::memory_order_relaxed); return true; } void swap(FakeCheckCheckCompletion& x){}; @@ -88,15 +88,15 @@ struct MemoryCheck { // // Really, copy constructor should be deletable, but CCheckQueue breaks // if it is deleted because of internal push_back. - fake_allocated_memory += b; + fake_allocated_memory.fetch_add(b, std::memory_order_relaxed); }; MemoryCheck(bool b_) : b(b_) { - fake_allocated_memory += b; + fake_allocated_memory.fetch_add(b, std::memory_order_relaxed); }; - ~MemoryCheck(){ - fake_allocated_memory -= b; - + ~MemoryCheck() + { + fake_allocated_memory.fetch_sub(b, std::memory_order_relaxed); }; void swap(MemoryCheck& x) { std::swap(b, x.b); }; }; @@ -117,9 +117,9 @@ struct FrozenCleanupCheck { { if (should_freeze) { std::unique_lock<std::mutex> l(m); - nFrozen = 1; + nFrozen.store(1, std::memory_order_relaxed); cv.notify_one(); - cv.wait(l, []{ return nFrozen == 0;}); + cv.wait(l, []{ return nFrozen.load(std::memory_order_relaxed) == 0;}); } } void swap(FrozenCleanupCheck& x){std::swap(should_freeze, x.should_freeze);}; @@ -262,7 +262,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) control.Add(vChecks); } bool r =control.Wait(); - BOOST_REQUIRE(r || end_fails); + BOOST_REQUIRE(r != end_fails); } } tg.interrupt_all(); @@ -337,7 +337,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) tg.join_all(); } -// Test that a new verification cannot occur until all checks +// Test that a new verification cannot occur until all checks // have been destructed BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) { @@ -361,11 +361,14 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) std::unique_lock<std::mutex> l(FrozenCleanupCheck::m); // Wait until the queue has finished all jobs and frozen FrozenCleanupCheck::cv.wait(l, [](){return FrozenCleanupCheck::nFrozen == 1;}); - // Try to get control of the queue a bunch of times - for (auto x = 0; x < 100 && !fails; ++x) { - fails = queue->ControlMutex.try_lock(); - } - // Unfreeze + } + // Try to get control of the queue a bunch of times + for (auto x = 0; x < 100 && !fails; ++x) { + fails = queue->ControlMutex.try_lock(); + } + { + // Unfreeze (we need lock n case of spurious wakeup) + std::unique_lock<std::mutex> l(FrozenCleanupCheck::m); FrozenCleanupCheck::nFrozen = 0; } // Awaken frozen destructor diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index dc358bff95..36e271295a 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "coins.h" -#include "script/standard.h" -#include "uint256.h" -#include "undo.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" -#include "validation.h" -#include "consensus/validation.h" +#include <coins.h> +#include <script/standard.h> +#include <uint256.h> +#include <undo.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> +#include <validation.h> +#include <consensus/validation.h> #include <vector> #include <map> @@ -81,8 +81,8 @@ public: // Manually recompute the dynamic usage of the whole data, and compare it. size_t ret = memusage::DynamicUsage(cacheCoins); size_t count = 0; - for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - ret += it->second.coin.DynamicMemoryUsage(); + for (const auto& entry : cacheCoins) { + ret += entry.second.coin.DynamicMemoryUsage(); ++count; } BOOST_CHECK_EQUAL(GetCacheSize(), count); @@ -189,15 +189,15 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Once every 1000 iterations and at the end, verify the full cache. if (InsecureRandRange(1000) == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (auto it = result.begin(); it != result.end(); it++) { - bool have = stack.back()->HaveCoin(it->first); - const Coin& coin = stack.back()->AccessCoin(it->first); + for (const auto& entry : result) { + bool have = stack.back()->HaveCoin(entry.first); + const Coin& coin = stack.back()->AccessCoin(entry.first); BOOST_CHECK(have == !coin.IsSpent()); - BOOST_CHECK(coin == it->second); + BOOST_CHECK(coin == entry.second); if (coin.IsSpent()) { missed_an_entry = true; } else { - BOOST_CHECK(stack.back()->HaveCoinInCache(it->first)); + BOOST_CHECK(stack.back()->HaveCoinInCache(entry.first)); found_an_entry = true; } } @@ -420,11 +420,11 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // Once every 1000 iterations and at the end, verify the full cache. if (InsecureRandRange(1000) == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (auto it = result.begin(); it != result.end(); it++) { - bool have = stack.back()->HaveCoin(it->first); - const Coin& coin = stack.back()->AccessCoin(it->first); + for (const auto& entry : result) { + bool have = stack.back()->HaveCoin(entry.first); + const Coin& coin = stack.back()->AccessCoin(entry.first); BOOST_CHECK(have == !coin.IsSpent()); - BOOST_CHECK(coin == it->second); + BOOST_CHECK(coin == entry.second); } } diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 35e4458bba..3c26013622 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2015 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "compressor.h" -#include "util.h" -#include "test/test_bitcoin.h" +#include <compressor.h> +#include <util.h> +#include <test/test_bitcoin.h> #include <stdint.h> diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index c748b2448c..cdfc664d56 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -1,18 +1,18 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypto/aes.h" -#include "crypto/chacha20.h" -#include "crypto/ripemd160.h" -#include "crypto/sha1.h" -#include "crypto/sha256.h" -#include "crypto/sha512.h" -#include "crypto/hmac_sha256.h" -#include "crypto/hmac_sha512.h" -#include "random.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <crypto/aes.h> +#include <crypto/chacha20.h> +#include <crypto/ripemd160.h> +#include <crypto/sha1.h> +#include <crypto/sha256.h> +#include <crypto/sha512.h> +#include <crypto/hmac_sha256.h> +#include <crypto/hmac_sha512.h> +#include <random.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <vector> @@ -340,6 +340,22 @@ BOOST_AUTO_TEST_CASE(hmac_sha256_testvectors) { "647320746f20626520686173686564206265666f7265206265696e6720757365" "642062792074686520484d414320616c676f726974686d2e", "9b09ffa71b942fcb27635fbcd5b0e944bfdc63644f0713938a7f51535c3a35e2"); + // Test case with key length 63 bytes. + TestHMACSHA256("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a6566", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "9de4b546756c83516720a4ad7fe7bdbeac4298c6fdd82b15f895a6d10b0769a6"); + // Test case with key length 64 bytes. + TestHMACSHA256("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "528c609a4c9254c274585334946b7c2661bad8f1fc406b20f6892478d19163dd"); + // Test case with key length 65 bytes. + TestHMACSHA256("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "d06af337f359a2330deffb8e3cbe4b5b7aa8ca1f208528cdbd245d5dc63c4483"); } BOOST_AUTO_TEST_CASE(hmac_sha512_testvectors) { @@ -383,6 +399,31 @@ BOOST_AUTO_TEST_CASE(hmac_sha512_testvectors) { "642062792074686520484d414320616c676f726974686d2e", "e37b6a775dc87dbaa4dfa9f96e5e3ffddebd71f8867289865df5a32d20cdc944" "b6022cac3c4982b10d5eeb55c3e4de15134676fb6de0446065c97440fa8c6a58"); + // Test case with key length 127 bytes. + TestHMACSHA512("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a6566", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "267424dfb8eeb999f3e5ec39a4fe9fd14c923e6187e0897063e5c9e02b2e624a" + "c04413e762977df71a9fb5d562b37f89dfdfb930fce2ed1fa783bbc2a203d80e"); + // Test case with key length 128 bytes. + TestHMACSHA512("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "43aaac07bb1dd97c82c04df921f83b16a68d76815cd1a30d3455ad43a3d80484" + "2bb35462be42cc2e4b5902de4d204c1c66d93b47d1383e3e13a3788687d61258"); + // Test case with key length 129 bytes. + TestHMACSHA512("4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a6566654a6566654a6566654a6566654a6566654a6566654a6566654a656665" + "4a", + "7768617420646f2079612077616e7420666f72206e6f7468696e673f", + "0b273325191cfc1b4b71d5075c8fcad67696309d292b1dad2cd23983a35feb8e" + "fb29795e79f2ef27f68cb1e16d76178c307a67beaad9456fac5fdffeadb16e2c"); } BOOST_AUTO_TEST_CASE(aes_testvectors) { diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 1004482224..51ebfc3800 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> -#include "cuckoocache.h" -#include "script/sigcache.h" -#include "test/test_bitcoin.h" -#include "random.h" +#include <cuckoocache.h> +#include <script/sigcache.h> +#include <test/test_bitcoin.h> +#include <random.h> #include <thread> /** Test Suite for CuckooCache diff --git a/src/test/data/script_tests.json b/src/test/data/script_tests.json index 698e898231..63f43c0fc6 100644 --- a/src/test/data/script_tests.json +++ b/src/test/data/script_tests.json @@ -862,8 +862,6 @@ ["Ensure 100% coverage of discouraged NOPS"], ["1", "NOP1", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], -["1", "CHECKLOCKTIMEVERIFY", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], -["1", "CHECKSEQUENCEVERIFY", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], ["1", "NOP4", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], ["1", "NOP5", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], ["1", "NOP6", "P2SH,DISCOURAGE_UPGRADABLE_NOPS", "DISCOURAGE_UPGRADABLE_NOPS"], diff --git a/src/test/data/tx_valid.json b/src/test/data/tx_valid.json index ad74b7cf1b..a845083636 100644 --- a/src/test/data/tx_valid.json +++ b/src/test/data/tx_valid.json @@ -56,6 +56,12 @@ [[["0000000000000000000000000000000000000000000000000000000000000200", 0, "1"], ["0000000000000000000000000000000000000000000000000000000000000100", 0, "DUP HASH160 0x14 0xe52b482f2faa8ecbf0db344f93c84ac908557f33 EQUALVERIFY CHECKSIG"]], "01000000020002000000000000000000000000000000000000000000000000000000000000000000000151ffffffff0001000000000000000000000000000000000000000000000000000000000000000000006b483045022100c9cdd08798a28af9d1baf44a6c77bcc7e279f47dc487c8c899911bc48feaffcc0220503c5c50ae3998a733263c5c0f7061b483e2b56c4c41b456e7d2f5a78a74c077032102d5c25adb51b61339d2b05315791e21bbe80ea470a49db0135720983c905aace0ffffffff010000000000000000015100000000", "P2SH"], +["The following tests SIGHASH_SINGLE|SIGHASHANYONECANPAY inputs"], +[[["437a1002eb125dec0f93f635763e0ae45f28ff8e81d82945753d0107611cd390", 1, "DUP HASH160 0x14 0x383fb81cb0a3fc724b5e08cf8bbd404336d711f6 EQUALVERIFY CHECKSIG"], + ["2d48d32ccad087bcda0ac5b31555bd58d1d2568184cbc8e752dd2be2684af03f", 1, "DUP HASH160 0x14 0x275ec2a233e5b23d43fa19e7bf9beb0cb3996117 EQUALVERIFY CHECKSIG"], + ["c76168ef1a272a4f176e55e73157ecfce040cfad16a5272f6296eb7089dca846", 1, "DUP HASH160 0x14 0x34fea2c5a75414fd945273ae2d029ce1f28dafcf EQUALVERIFY CHECKSIG"]], +"010000000390d31c6107013d754529d8818eff285fe40a3e7635f6930fec5d12eb02107a43010000006b483045022100f40815ae3c81a0dd851cc8d376d6fd226c88416671346a9033468cca2cdcc6c202204f764623903e6c4bed1b734b75d82c40f1725e4471a55ad4f51218f86130ac038321033d710ab45bb54ac99618ad23b3c1da661631aa25f23bfe9d22b41876f1d46e4effffffff3ff04a68e22bdd52e7c8cb848156d2d158bd5515b3c50adabc87d0ca2cd3482d010000006a4730440220598d263c107004008e9e26baa1e770be30fd31ee55ded1898f7c00da05a75977022045536bead322ca246779698b9c3df3003377090f41afeca7fb2ce9e328ec4af2832102b738b531def73020bd637f32935924cc88549c8206976226d968edd3a42fc2d7ffffffff46a8dc8970eb96622f27a516adcf40e0fcec5731e7556e174f2a271aef6861c7010000006b483045022100c5b90a777a9fdc90c208dbef7290d1fc1be651f47151ee4ccff646872a454cf90220640cfbc4550446968fbbe9d12528f3adf7d87b31541569c59e790db8a220482583210391332546e22bbe8fe3af54addfad6f8b83d05fa4f5e047593d4c07ae938795beffffffff028036be26000000001976a914ddfb29efad43a667465ac59ff14dc6442a1adfca88ac3d5cba01000000001976a914b64dde7a505a13ca986c40e86e984a8dc81368b688ac00000000", "P2SH"], + ["An invalid P2SH Transaction"], [[["0000000000000000000000000000000000000000000000000000000000000100", 0, "HASH160 0x14 0x7a052c840ba73af26755de42cf01cc9e0a49fef0 EQUAL"]], "010000000100010000000000000000000000000000000000000000000000000000000000000000000009085768617420697320ffffffff010000000000000000015100000000", "NONE"], diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index efddafe17e..754a86344f 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "dbwrapper.h" -#include "uint256.h" -#include "random.h" -#include "test/test_bitcoin.h" +#include <dbwrapper.h> +#include <uint256.h> +#include <random.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> @@ -125,7 +125,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) create_directories(ph); // Set up a non-obfuscated wrapper to write some initial data. - CDBWrapper* dbw = new CDBWrapper(ph, (1 << 10), false, false, false); + std::unique_ptr<CDBWrapper> dbw = MakeUnique<CDBWrapper>(ph, (1 << 10), false, false, false); char key = 'k'; uint256 in = InsecureRand256(); uint256 res; @@ -135,8 +135,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate) BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); // Call the destructor to free leveldb LOCK - delete dbw; - dbw = nullptr; + dbw.reset(); // Now, set up another wrapper that wants to obfuscate the same directory CDBWrapper odbw(ph, (1 << 10), false, false, true); @@ -167,7 +166,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) create_directories(ph); // Set up a non-obfuscated wrapper to write some initial data. - CDBWrapper* dbw = new CDBWrapper(ph, (1 << 10), false, false, false); + std::unique_ptr<CDBWrapper> dbw = MakeUnique<CDBWrapper>(ph, (1 << 10), false, false, false); char key = 'k'; uint256 in = InsecureRand256(); uint256 res; @@ -177,8 +176,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex) BOOST_CHECK_EQUAL(res.ToString(), in.ToString()); // Call the destructor to free leveldb LOCK - delete dbw; - dbw = nullptr; + dbw.reset(); // Simulate a -reindex by wiping the existing data store CDBWrapper odbw(ph, (1 << 10), false, true, true); diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 40f0ecd5f1..cd603b7f58 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2015 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "util.h" -#include "test/test_bitcoin.h" +#include <util.h> +#include <test/test_bitcoin.h> #include <string> #include <vector> diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 05b6b3b1e6..0de0a17904 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2013-2016 The Bitcoin Core developers +// Copyright (c) 2013-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "hash.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <hash.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 91c0175412..55ee1ecf6b 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2015 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "key.h" +#include <key.h> -#include "base58.h" -#include "script/script.h" -#include "uint256.h" -#include "util.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" +#include <base58.h> +#include <script/script.h> +#include <uint256.h> +#include <util.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> #include <string> #include <vector> diff --git a/src/test/limitedmap_tests.cpp b/src/test/limitedmap_tests.cpp index b071ab117b..a4bd63cdef 100644 --- a/src/test/limitedmap_tests.cpp +++ b/src/test/limitedmap_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "limitedmap.h" +#include <limitedmap.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/main_tests.cpp b/src/test/main_tests.cpp index 656aec606b..a833a5cb1e 100644 --- a/src/test/main_tests.cpp +++ b/src/test/main_tests.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chainparams.h" -#include "validation.h" -#include "net.h" +#include <chainparams.h> +#include <validation.h> +#include <net.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/signals2/signal.hpp> #include <boost/test/unit_test.hpp> diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 116210a297..e5afeddf26 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "policy/policy.h" -#include "txmempool.h" -#include "util.h" +#include <policy/policy.h> +#include <txmempool.h> +#include <util.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> #include <list> @@ -165,6 +165,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) sortedOrder[2] = tx1.GetHash().ToString(); // 10000 sortedOrder[3] = tx4.GetHash().ToString(); // 15000 sortedOrder[4] = tx2.GetHash().ToString(); // 20000 + LOCK(pool.cs); CheckSort<descendant_score>(pool, sortedOrder); /* low fee but with high fee child */ @@ -286,35 +287,6 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx()); pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx()); - /* Now check the sort on the mining score index. - * Final order should be: - * - * tx7 (2M) - * tx2 (20k) - * tx4 (15000) - * tx1/tx5 (10000) - * tx3/6 (0) - * (Ties resolved by hash) - */ - sortedOrder.clear(); - sortedOrder.push_back(tx7.GetHash().ToString()); - sortedOrder.push_back(tx2.GetHash().ToString()); - sortedOrder.push_back(tx4.GetHash().ToString()); - if (tx1.GetHash() < tx5.GetHash()) { - sortedOrder.push_back(tx5.GetHash().ToString()); - sortedOrder.push_back(tx1.GetHash().ToString()); - } else { - sortedOrder.push_back(tx1.GetHash().ToString()); - sortedOrder.push_back(tx5.GetHash().ToString()); - } - if (tx3.GetHash() < tx6.GetHash()) { - sortedOrder.push_back(tx6.GetHash().ToString()); - sortedOrder.push_back(tx3.GetHash().ToString()); - } else { - sortedOrder.push_back(tx3.GetHash().ToString()); - sortedOrder.push_back(tx6.GetHash().ToString()); - } - CheckSort<mining_score>(pool, sortedOrder); } BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) @@ -375,6 +347,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) } sortedOrder[4] = tx3.GetHash().ToString(); // 0 + LOCK(pool.cs); CheckSort<ancestor_score>(pool, sortedOrder); /* low fee parent with high fee child */ diff --git a/src/test/merkle_tests.cpp b/src/test/merkle_tests.cpp index 1a1cf4399c..72a2672352 100644 --- a/src/test/merkle_tests.cpp +++ b/src/test/merkle_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/merkle.h" -#include "test/test_bitcoin.h" +#include <consensus/merkle.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/merkleblock_tests.cpp b/src/test/merkleblock_tests.cpp index 3e66c6f2c6..37a1a84136 100644 --- a/src/test/merkleblock_tests.cpp +++ b/src/test/merkleblock_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "merkleblock.h" -#include "uint256.h" -#include "test/test_bitcoin.h" +#include <merkleblock.h> +#include <uint256.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 41e0626eb9..d9f6772c2d 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -1,24 +1,24 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chainparams.h" -#include "coins.h" -#include "consensus/consensus.h" -#include "consensus/merkle.h" -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "validation.h" -#include "miner.h" -#include "policy/policy.h" -#include "pubkey.h" -#include "script/standard.h" -#include "txmempool.h" -#include "uint256.h" -#include "util.h" -#include "utilstrencodings.h" - -#include "test/test_bitcoin.h" +#include <chainparams.h> +#include <coins.h> +#include <consensus/consensus.h> +#include <consensus/merkle.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <validation.h> +#include <miner.h> +#include <policy/policy.h> +#include <pubkey.h> +#include <script/standard.h> +#include <txmempool.h> +#include <uint256.h> +#include <util.h> +#include <utilstrencodings.h> + +#include <test/test_bitcoin.h> #include <memory> @@ -26,6 +26,17 @@ BOOST_FIXTURE_TEST_SUITE(miner_tests, TestingSetup) +// BOOST_CHECK_EXCEPTION predicates to check the specific validation error +class HasReason { +public: + HasReason(const std::string& reason) : m_reason(reason) {} + bool operator() (const std::runtime_error& e) const { + return std::string(e.what()).find(m_reason) != std::string::npos; + }; +private: + const std::string m_reason; +}; + static CFeeRate blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); static BlockAssembler AssemblerForTest(const CChainParams& params) { @@ -205,7 +216,6 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) entry.nFee = 11; entry.nHeight = 11; - LOCK(cs_main); fCheckpointsEnabled = false; // Simple block creation, nothing special yet: @@ -218,27 +228,32 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) for (unsigned int i = 0; i < sizeof(blockinfo)/sizeof(*blockinfo); ++i) { CBlock *pblock = &pblocktemplate->block; // pointer for convenience - pblock->nVersion = 1; - pblock->nTime = chainActive.Tip()->GetMedianTimePast()+1; - CMutableTransaction txCoinbase(*pblock->vtx[0]); - txCoinbase.nVersion = 1; - txCoinbase.vin[0].scriptSig = CScript(); - txCoinbase.vin[0].scriptSig.push_back(blockinfo[i].extranonce); - txCoinbase.vin[0].scriptSig.push_back(chainActive.Height()); - txCoinbase.vout.resize(1); // Ignore the (optional) segwit commitment added by CreateNewBlock (as the hardcoded nonces don't account for this) - txCoinbase.vout[0].scriptPubKey = CScript(); - pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); - if (txFirst.size() == 0) - baseheight = chainActive.Height(); - if (txFirst.size() < 4) - txFirst.push_back(pblock->vtx[0]); - pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); - pblock->nNonce = blockinfo[i].nonce; + { + LOCK(cs_main); + pblock->nVersion = 1; + pblock->nTime = chainActive.Tip()->GetMedianTimePast()+1; + CMutableTransaction txCoinbase(*pblock->vtx[0]); + txCoinbase.nVersion = 1; + txCoinbase.vin[0].scriptSig = CScript(); + txCoinbase.vin[0].scriptSig.push_back(blockinfo[i].extranonce); + txCoinbase.vin[0].scriptSig.push_back(chainActive.Height()); + txCoinbase.vout.resize(1); // Ignore the (optional) segwit commitment added by CreateNewBlock (as the hardcoded nonces don't account for this) + txCoinbase.vout[0].scriptPubKey = CScript(); + pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); + if (txFirst.size() == 0) + baseheight = chainActive.Height(); + if (txFirst.size() < 4) + txFirst.push_back(pblock->vtx[0]); + pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); + pblock->nNonce = blockinfo[i].nonce; + } std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); BOOST_CHECK(ProcessNewBlock(chainparams, shared_pblock, true, nullptr)); pblock->hashPrevBlock = pblock->GetHash(); } + LOCK(cs_main); + // Just to make sure we can still make simple blocks BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); @@ -264,7 +279,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } - BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-blk-sigops")); mempool.clear(); tx.vin[0].prevout.hash = txFirst[0]->GetHash(); @@ -304,7 +320,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) // orphan in mempool, template creation fails hash = tx.GetHash(); mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).FromTx(tx)); - BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); mempool.clear(); // child with higher feerate than parent @@ -332,24 +348,8 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) hash = tx.GetHash(); // give it a fee so it'll get mined mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); - mempool.clear(); - - // invalid (pre-p2sh) txn in mempool, template creation fails - tx.vin[0].prevout.hash = txFirst[0]->GetHash(); - tx.vin[0].prevout.n = 0; - tx.vin[0].scriptSig = CScript() << OP_1; - tx.vout[0].nValue = BLOCKSUBSIDY-LOWFEE; - script = CScript() << OP_0; - tx.vout[0].scriptPubKey = GetScriptForDestination(CScriptID(script)); - hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - tx.vin[0].prevout.hash = hash; - tx.vin[0].scriptSig = CScript() << std::vector<unsigned char>(script.begin(), script.end()); - tx.vout[0].nValue -= LOWFEE; - hash = tx.GetHash(); - mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); - BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + // Should throw bad-cb-multiple + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-cb-multiple")); mempool.clear(); // double spend txn pair in mempool, template creation fails @@ -362,7 +362,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) tx.vout[0].scriptPubKey = CScript() << OP_2; hash = tx.GetHash(); mempool.addUnchecked(hash, entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); - BOOST_CHECK_THROW(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error); + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); mempool.clear(); // subsidy changing @@ -391,6 +391,25 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) chainActive.SetTip(next); } BOOST_CHECK(pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey)); + + // invalid p2sh txn in mempool, template creation fails + tx.vin[0].prevout.hash = txFirst[0]->GetHash(); + tx.vin[0].prevout.n = 0; + tx.vin[0].scriptSig = CScript() << OP_1; + tx.vout[0].nValue = BLOCKSUBSIDY-LOWFEE; + script = CScript() << OP_0; + tx.vout[0].scriptPubKey = GetScriptForDestination(CScriptID(script)); + hash = tx.GetHash(); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx.vin[0].prevout.hash = hash; + tx.vin[0].scriptSig = CScript() << std::vector<unsigned char>(script.begin(), script.end()); + tx.vout[0].nValue -= LOWFEE; + hash = tx.GetHash(); + mempool.addUnchecked(hash, entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + // Should throw block-validation-failed + BOOST_CHECK_EXCEPTION(AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("block-validation-failed")); + mempool.clear(); + // Delete the dummy blocks again. while (chainActive.Tip()->nHeight > nHeight) { CBlockIndex* del = chainActive.Tip(); diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index de7f3b48f5..72d9d3d75c 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "key.h" -#include "keystore.h" -#include "policy/policy.h" -#include "script/script.h" -#include "script/script_error.h" -#include "script/interpreter.h" -#include "script/sign.h" -#include "script/ismine.h" -#include "uint256.h" -#include "test/test_bitcoin.h" +#include <key.h> +#include <keystore.h> +#include <policy/policy.h> +#include <script/script.h> +#include <script/script_error.h> +#include <script/interpreter.h> +#include <script/sign.h> +#include <script/ismine.h> +#include <uint256.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 31b05d868b..ca57f58905 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "addrman.h" -#include "test/test_bitcoin.h" +#include <addrman.h> +#include <test/test_bitcoin.h> #include <string> #include <boost/test/unit_test.hpp> -#include "hash.h" -#include "serialize.h" -#include "streams.h" -#include "net.h" -#include "netbase.h" -#include "chainparams.h" -#include "util.h" +#include <hash.h> +#include <serialize.h> +#include <streams.h> +#include <net.h> +#include <netbase.h> +#include <chainparams.h> +#include <util.h> class CAddrManSerializationMock : public CAddrMan { diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 1baf7643e5..027214e512 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "netbase.h" -#include "test/test_bitcoin.h" -#include "utilstrencodings.h" +#include <netbase.h> +#include <test/test_bitcoin.h> +#include <utilstrencodings.h> #include <string> diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index c1d216d094..4825c92296 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/merkle.h" -#include "merkleblock.h" -#include "serialize.h" -#include "streams.h" -#include "uint256.h" -#include "arith_uint256.h" -#include "version.h" -#include "test/test_bitcoin.h" +#include <consensus/merkle.h> +#include <merkleblock.h> +#include <serialize.h> +#include <streams.h> +#include <uint256.h> +#include <arith_uint256.h> +#include <version.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index fd8f7191f4..8d288ec993 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "policy/policy.h" -#include "policy/fees.h" -#include "txmempool.h" -#include "uint256.h" -#include "util.h" +#include <policy/policy.h> +#include <policy/fees.h> +#include <txmempool.h> +#include <uint256.h> +#include <util.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index b13f2625aa..26b2f5d0d7 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" -#include "chainparams.h" -#include "pow.h" -#include "random.h" -#include "util.h" -#include "test/test_bitcoin.h" +#include <chain.h> +#include <chainparams.h> +#include <pow.h> +#include <random.h> +#include <util.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 841282873f..db9162c0db 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <vector> -#include "prevector.h" +#include <prevector.h> -#include "reverse_iterator.h" -#include "serialize.h" -#include "streams.h" +#include <reverse_iterator.h> +#include <serialize.h> +#include <streams.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/raii_event_tests.cpp b/src/test/raii_event_tests.cpp index 0d541ec7d4..7bc81c7f64 100644 --- a/src/test/raii_event_tests.cpp +++ b/src/test/raii_event_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,9 +10,9 @@ #include <map> #include <stdlib.h> -#include "support/events.h" +#include <support/events.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 132e190051..1ca5a53d72 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "random.h" +#include <random.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp index 00dc47e13e..0a91c65afa 100644 --- a/src/test/reverselock_tests.cpp +++ b/src/test/reverselock_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "reverselock.h" -#include "test/test_bitcoin.h" +#include <reverselock.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index c6643be7a7..ed86413b2f 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "rpc/server.h" -#include "rpc/client.h" +#include <rpc/server.h> +#include <rpc/client.h> -#include "base58.h" -#include "core_io.h" -#include "netbase.h" +#include <base58.h> +#include <core_io.h> +#include <netbase.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/algorithm/string.hpp> #include <boost/test/unit_test.hpp> @@ -65,7 +65,9 @@ BOOST_AUTO_TEST_CASE(rpc_rawparams) BOOST_CHECK_EQUAL(find_value(r.get_obj(), "size").get_int(), 193); BOOST_CHECK_EQUAL(find_value(r.get_obj(), "version").get_int(), 1); BOOST_CHECK_EQUAL(find_value(r.get_obj(), "locktime").get_int(), 0); - BOOST_CHECK_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx+" extra"), std::runtime_error); + BOOST_CHECK_THROW(CallRPC(std::string("decoderawtransaction ")+rawtx+" extra"), std::runtime_error); + BOOST_CHECK_NO_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx+" false")); + BOOST_CHECK_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx+" false extra"), std::runtime_error); BOOST_CHECK_THROW(CallRPC("signrawtransaction"), std::runtime_error); BOOST_CHECK_THROW(CallRPC("signrawtransaction null"), std::runtime_error); diff --git a/src/test/sanity_tests.cpp b/src/test/sanity_tests.cpp index 51f9e9f39f..e97ad2ee6a 100644 --- a/src/test/sanity_tests.cpp +++ b/src/test/sanity_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2015 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "compat/sanity.h" -#include "key.h" -#include "test/test_bitcoin.h" +#include <compat/sanity.h> +#include <key.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 1de865776e..760f933abc 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "random.h" -#include "scheduler.h" +#include <random.h> +#include <scheduler.h> -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> #include <boost/bind.hpp> #include <boost/thread.hpp> diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index 58aa32c969..63d211dd97 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -1,18 +1,18 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/tx_verify.h" -#include "core_io.h" -#include "key.h" -#include "keystore.h" -#include "validation.h" -#include "policy/policy.h" -#include "script/script.h" -#include "script/script_error.h" -#include "script/sign.h" -#include "script/ismine.h" -#include "test/test_bitcoin.h" +#include <consensus/tx_verify.h> +#include <core_io.h> +#include <key.h> +#include <keystore.h> +#include <validation.h> +#include <policy/policy.h> +#include <script/script.h> +#include <script/script_error.h> +#include <script/sign.h> +#include <script/ismine.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index bd2d9ed115..cd30fbeda7 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "key.h" -#include "keystore.h" -#include "script/ismine.h" -#include "script/script.h" -#include "script/script_error.h" -#include "script/standard.h" -#include "test/test_bitcoin.h" +#include <key.h> +#include <keystore.h> +#include <script/ismine.h> +#include <script/script.h> +#include <script/script_error.h> +#include <script/standard.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> @@ -508,12 +508,7 @@ BOOST_AUTO_TEST_CASE(script_standard_IsMine) scriptPubKey.clear(); scriptPubKey << OP_0 << ToByteVector(pubkeys[0].GetID()); - // Keystore has key, but no P2SH redeemScript - result = IsMine(keystore, scriptPubKey, isInvalid); - BOOST_CHECK_EQUAL(result, ISMINE_NO); - BOOST_CHECK(!isInvalid); - - // Keystore has key and P2SH redeemScript + // Keystore implicitly has key and P2SH redeemScript keystore.AddCScript(scriptPubKey); result = IsMine(keystore, scriptPubKey, isInvalid); BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 011a5db795..c7a497f3a7 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1,22 +1,22 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "data/script_tests.json.h" +#include <test/data/script_tests.json.h> -#include "core_io.h" -#include "key.h" -#include "keystore.h" -#include "script/script.h" -#include "script/script_error.h" -#include "script/sign.h" -#include "util.h" -#include "utilstrencodings.h" -#include "test/test_bitcoin.h" -#include "rpc/server.h" +#include <core_io.h> +#include <key.h> +#include <keystore.h> +#include <script/script.h> +#include <script/script_error.h> +#include <script/sign.h> +#include <util.h> +#include <utilstrencodings.h> +#include <test/test_bitcoin.h> +#include <rpc/server.h> #if defined(HAVE_CONSENSUS_LIB) -#include "script/bitcoinconsensus.h" +#include <script/bitcoinconsensus.h> #endif #include <fstream> @@ -166,6 +166,17 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript CMutableTransaction tx2 = tx; BOOST_CHECK_MESSAGE(VerifyScript(scriptSig, scriptPubKey, &scriptWitness, flags, MutableTransactionSignatureChecker(&tx, 0, txCredit.vout[0].nValue), &err) == expect, message); BOOST_CHECK_MESSAGE(err == scriptError, std::string(FormatScriptError(err)) + " where " + std::string(FormatScriptError((ScriptError_t)scriptError)) + " expected: " + message); + + // Verify that removing flags from a passing test or adding flags to a failing test does not change the result. + for (int i = 0; i < 16; ++i) { + int extra_flags = InsecureRandBits(16); + int combined_flags = expect ? (flags & ~extra_flags) : (flags | extra_flags); + // Weed out some invalid flag combinations. + if (combined_flags & SCRIPT_VERIFY_CLEANSTACK && ~combined_flags & (SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS)) continue; + if (combined_flags & SCRIPT_VERIFY_WITNESS && ~combined_flags & SCRIPT_VERIFY_P2SH) continue; + BOOST_CHECK_MESSAGE(VerifyScript(scriptSig, scriptPubKey, &scriptWitness, combined_flags, MutableTransactionSignatureChecker(&tx, 0, txCredit.vout[0].nValue), &err) == expect, message + strprintf(" (with flags %x)", combined_flags)); + } + #if defined(HAVE_CONSENSUS_LIB) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << tx2; diff --git a/src/test/scriptnum10.h b/src/test/scriptnum10.h index 94dd58526c..9e88866cb0 100644 --- a/src/test/scriptnum10.h +++ b/src/test/scriptnum10.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ #include <stdint.h> #include <string> #include <vector> -#include "assert.h" +#include <assert.h> class scriptnum10_error : public std::runtime_error { diff --git a/src/test/scriptnum_tests.cpp b/src/test/scriptnum_tests.cpp index 280eb59ce8..cd6e27d248 100644 --- a/src/test/scriptnum_tests.cpp +++ b/src/test/scriptnum_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2015 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "scriptnum10.h" -#include "script/script.h" -#include "test/test_bitcoin.h" +#include <test/scriptnum10.h> +#include <script/script.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> #include <limits.h> diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 9661a66514..4595519435 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "serialize.h" -#include "streams.h" -#include "hash.h" -#include "test/test_bitcoin.h" +#include <serialize.h> +#include <streams.h> +#include <hash.h> +#include <test/test_bitcoin.h> #include <stdint.h> diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index ecbdf57788..32cd3a50b0 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -1,19 +1,19 @@ -// Copyright (c) 2013-2016 The Bitcoin Core developers +// Copyright (c) 2013-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "data/sighash.json.h" -#include "hash.h" -#include "script/interpreter.h" -#include "script/script.h" -#include "serialize.h" -#include "streams.h" -#include "test/test_bitcoin.h" -#include "util.h" -#include "utilstrencodings.h" -#include "version.h" +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <test/data/sighash.json.h> +#include <hash.h> +#include <script/interpreter.h> +#include <script/script.h> +#include <serialize.h> +#include <streams.h> +#include <test/test_bitcoin.h> +#include <util.h> +#include <utilstrencodings.h> +#include <version.h> #include <iostream> diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index d3b8b07228..b4d8a2419e 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "pubkey.h" -#include "key.h" -#include "script/script.h" -#include "script/standard.h" -#include "uint256.h" -#include "test/test_bitcoin.h" +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <pubkey.h> +#include <key.h> +#include <script/script.h> +#include <script/standard.h> +#include <uint256.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/skiplist_tests.cpp b/src/test/skiplist_tests.cpp index 164cbd873f..25c7e54808 100644 --- a/src/test/skiplist_tests.cpp +++ b/src/test/skiplist_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" -#include "util.h" -#include "test/test_bitcoin.h" +#include <chain.h> +#include <util.h> +#include <test/test_bitcoin.h> #include <vector> diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index af2a152aa5..1108dab584 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "streams.h" -#include "support/allocators/zeroafterfree.h" -#include "test/test_bitcoin.h" +#include <streams.h> +#include <support/allocators/zeroafterfree.h> +#include <test/test_bitcoin.h> #include <boost/assign/std/vector.hpp> // for 'operator+=()' #include <boost/test/unit_test.hpp> diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 79bc48a118..bdd44489f4 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -1,30 +1,36 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "test_bitcoin.h" - -#include "chainparams.h" -#include "consensus/consensus.h" -#include "consensus/validation.h" -#include "crypto/sha256.h" -#include "fs.h" -#include "key.h" -#include "validation.h" -#include "miner.h" -#include "net_processing.h" -#include "pubkey.h" -#include "random.h" -#include "txdb.h" -#include "txmempool.h" -#include "ui_interface.h" -#include "streams.h" -#include "rpc/server.h" -#include "rpc/register.h" -#include "script/sigcache.h" +#include <test/test_bitcoin.h> + +#include <chainparams.h> +#include <consensus/consensus.h> +#include <consensus/validation.h> +#include <crypto/sha256.h> +#include <validation.h> +#include <miner.h> +#include <net_processing.h> +#include <ui_interface.h> +#include <streams.h> +#include <rpc/server.h> +#include <rpc/register.h> +#include <script/sigcache.h> #include <memory> +void CConnmanTest::AddNode(CNode& node) +{ + LOCK(g_connman->cs_vNodes); + g_connman->vNodes.push_back(&node); +} + +void CConnmanTest::ClearNodes() +{ + LOCK(g_connman->cs_vNodes); + g_connman->vNodes.clear(); +} + uint256 insecure_rand_seed = GetRandHash(); FastRandomContext insecure_rand_ctx(insecure_rand_seed); @@ -63,15 +69,15 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha fs::create_directories(pathTemp); gArgs.ForceSetArg("-datadir", pathTemp.string()); - // Note that because we don't bother running a scheduler thread here, - // callbacks via CValidationInterface are unreliable, but that's OK, - // our unit tests aren't testing multiple parts of the code at once. + // We have to run a scheduler thread to prevent ActivateBestChain + // from blocking due to queue overrun. + threadGroup.create_thread(boost::bind(&CScheduler::serviceQueue, &scheduler)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); mempool.setSanityCheck(1.0); - pblocktree = new CBlockTreeDB(1 << 20, true); - pcoinsdbview = new CCoinsViewDB(1 << 23, true); - pcoinsTip = new CCoinsViewCache(pcoinsdbview); + pblocktree.reset(new CBlockTreeDB(1 << 20, true)); + pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); + pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); if (!LoadGenesisBlock(chainparams)) { throw std::runtime_error("LoadGenesisBlock failed."); } @@ -86,7 +92,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha threadGroup.create_thread(&ThreadScriptCheck); g_connman = std::unique_ptr<CConnman>(new CConnman(0x1337, 0x1337)); // Deterministic randomness for tests. connman = g_connman.get(); - peerLogic.reset(new PeerLogicValidation(connman)); + peerLogic.reset(new PeerLogicValidation(connman, scheduler)); } TestingSetup::~TestingSetup() @@ -98,14 +104,17 @@ TestingSetup::~TestingSetup() g_connman.reset(); peerLogic.reset(); UnloadBlockIndex(); - delete pcoinsTip; - delete pcoinsdbview; - delete pblocktree; + pcoinsTip.reset(); + pcoinsdbview.reset(); + pblocktree.reset(); fs::remove_all(pathTemp); } TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) { + // CreateAndProcessBlock() does not support building SegWit blocks, so don't activate in these tests. + // TODO: fix the code to support SegWit blocks. + UpdateVersionBitsParameters(Consensus::DEPLOYMENT_SEGWIT, 0, Consensus::BIP9Deployment::NO_TIMEOUT); // Generate a 100-block chain: coinbaseKey.MakeNewKey(true); CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; @@ -134,7 +143,10 @@ TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& block.vtx.push_back(MakeTransactionRef(tx)); // IncrementExtraNonce creates a valid coinbase and merkleRoot unsigned int extraNonce = 0; - IncrementExtraNonce(&block, chainActive.Tip(), extraNonce); + { + LOCK(cs_main); + IncrementExtraNonce(&block, chainActive.Tip(), extraNonce); + } while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index 2390aca342..944835cccf 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -1,18 +1,18 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TEST_TEST_BITCOIN_H #define BITCOIN_TEST_TEST_BITCOIN_H -#include "chainparamsbase.h" -#include "fs.h" -#include "key.h" -#include "pubkey.h" -#include "random.h" -#include "scheduler.h" -#include "txdb.h" -#include "txmempool.h" +#include <chainparamsbase.h> +#include <fs.h> +#include <key.h> +#include <pubkey.h> +#include <random.h> +#include <scheduler.h> +#include <txdb.h> +#include <txmempool.h> #include <boost/thread.hpp> @@ -49,9 +49,14 @@ struct BasicTestingSetup { * Included are data directory, coins database, script check threads setup. */ class CConnman; +class CNode; +struct CConnmanTest { + static void AddNode(CNode& node); + static void ClearNodes(); +}; + class PeerLogicValidation; struct TestingSetup: public BasicTestingSetup { - CCoinsViewDB *pcoinsdbview; fs::path pathTemp; boost::thread_group threadGroup; CConnman* connman; diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/test_bitcoin_fuzzy.cpp index 581ad2ffa0..aaba2095e0 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/test_bitcoin_fuzzy.cpp @@ -1,24 +1,25 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "consensus/merkle.h" -#include "primitives/block.h" -#include "script/script.h" -#include "addrman.h" -#include "chain.h" -#include "coins.h" -#include "compressor.h" -#include "net.h" -#include "protocol.h" -#include "streams.h" -#include "undo.h" -#include "version.h" -#include "pubkey.h" +#include <consensus/merkle.h> +#include <primitives/block.h> +#include <script/script.h> +#include <addrman.h> +#include <chain.h> +#include <coins.h> +#include <compressor.h> +#include <net.h> +#include <protocol.h> +#include <streams.h> +#include <undo.h> +#include <version.h> +#include <pubkey.h> +#include <blockencodings.h> #include <stdint.h> #include <unistd.h> @@ -45,6 +46,8 @@ enum TEST_ID { CBLOOMFILTER_DESERIALIZE, CDISKBLOCKINDEX_DESERIALIZE, CTXOUTCOMPRESSOR_DESERIALIZE, + BLOCKTRANSACTIONS_DESERIALIZE, + BLOCKTRANSACTIONSREQUEST_DESERIALIZE, TEST_ID_END }; @@ -245,6 +248,26 @@ int test_one_input(std::vector<uint8_t> buffer) { break; } + case BLOCKTRANSACTIONS_DESERIALIZE: + { + try + { + BlockTransactions bt; + ds >> bt; + } catch (const std::ios_base::failure& e) {return 0;} + + break; + } + case BLOCKTRANSACTIONSREQUEST_DESERIALIZE: + { + try + { + BlockTransactionsRequest btr; + ds >> btr; + } catch (const std::ios_base::failure& e) {return 0;} + + break; + } default: return 0; } diff --git a/src/test/test_bitcoin_main.cpp b/src/test/test_bitcoin_main.cpp index b556c953b9..64408e9c5b 100644 --- a/src/test/test_bitcoin_main.cpp +++ b/src/test/test_bitcoin_main.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #define BOOST_TEST_MODULE Bitcoin Test Suite -#include "net.h" +#include <net.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp index 34863fd9d0..5d93bc1efe 100644 --- a/src/test/timedata_tests.cpp +++ b/src/test/timedata_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2015 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // -#include "timedata.h" -#include "test/test_bitcoin.h" +#include <timedata.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/torcontrol_tests.cpp b/src/test/torcontrol_tests.cpp index b7affaacde..d0aa8659c2 100644 --- a/src/test/torcontrol_tests.cpp +++ b/src/test/torcontrol_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // -#include "test/test_bitcoin.h" -#include "torcontrol.cpp" +#include <test/test_bitcoin.h> +#include <torcontrol.cpp> #include <boost/test/unit_test.hpp> diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index cb6ab7cdbe..edfb35d155 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -1,25 +1,25 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "data/tx_invalid.json.h" -#include "data/tx_valid.json.h" -#include "test/test_bitcoin.h" - -#include "clientversion.h" -#include "checkqueue.h" -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "core_io.h" -#include "key.h" -#include "keystore.h" -#include "validation.h" -#include "policy/policy.h" -#include "script/script.h" -#include "script/sign.h" -#include "script/script_error.h" -#include "script/standard.h" -#include "utilstrencodings.h" +#include <test/data/tx_invalid.json.h> +#include <test/data/tx_valid.json.h> +#include <test/test_bitcoin.h> + +#include <clientversion.h> +#include <checkqueue.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <core_io.h> +#include <key.h> +#include <keystore.h> +#include <validation.h> +#include <policy/policy.h> +#include <script/script.h> +#include <script/sign.h> +#include <script/script_error.h> +#include <script/standard.h> +#include <utilstrencodings.h> #include <map> #include <string> diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp new file mode 100644 index 0000000000..2d1eb7b772 --- /dev/null +++ b/src/test/txvalidation_tests.cpp @@ -0,0 +1,61 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <validation.h> +#include <txmempool.h> +#include <amount.h> +#include <consensus/validation.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <test/test_bitcoin.h> + +#include <boost/test/unit_test.hpp> + + +BOOST_AUTO_TEST_SUITE(txvalidation_tests) + +/** + * Ensure that the mempool won't accept coinbase transactions. + */ +BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) +{ + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + CMutableTransaction coinbaseTx; + + coinbaseTx.nVersion = 1; + coinbaseTx.vin.resize(1); + coinbaseTx.vout.resize(1); + coinbaseTx.vin[0].scriptSig = CScript() << OP_11 << OP_EQUAL; + coinbaseTx.vout[0].nValue = 1 * CENT; + coinbaseTx.vout[0].scriptPubKey = scriptPubKey; + + assert(CTransaction(coinbaseTx).IsCoinBase()); + + CValidationState state; + + LOCK(cs_main); + + unsigned int initialPoolSize = mempool.size(); + + BOOST_CHECK_EQUAL( + false, + AcceptToMemoryPool(mempool, state, MakeTransactionRef(coinbaseTx), + nullptr /* pfMissingInputs */, + nullptr /* plTxnReplaced */, + true /* bypass_limits */, + 0 /* nAbsurdFee */)); + + // Check that the transaction hasn't been added to mempool. + BOOST_CHECK_EQUAL(mempool.size(), initialPoolSize); + + // Check that the validation state reflects the unsuccessful attempt. + BOOST_CHECK(state.IsInvalid()); + BOOST_CHECK_EQUAL(state.GetRejectReason(), "coinbase"); + + int nDoS; + BOOST_CHECK_EQUAL(state.IsInvalid(nDoS), true); + BOOST_CHECK_EQUAL(nDoS, 100); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 82ca93e7da..9ec9d6cba3 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/validation.h" -#include "key.h" -#include "validation.h" -#include "miner.h" -#include "pubkey.h" -#include "txmempool.h" -#include "random.h" -#include "script/standard.h" -#include "script/sign.h" -#include "test/test_bitcoin.h" -#include "utiltime.h" -#include "core_io.h" -#include "keystore.h" -#include "policy/policy.h" +#include <consensus/validation.h> +#include <key.h> +#include <validation.h> +#include <miner.h> +#include <pubkey.h> +#include <txmempool.h> +#include <random.h> +#include <script/standard.h> +#include <script/sign.h> +#include <test/test_bitcoin.h> +#include <utiltime.h> +#include <core_io.h> +#include <keystore.h> +#include <policy/policy.h> #include <boost/test/unit_test.hpp> @@ -102,7 +102,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // should fail. // Capture this interaction with the upgraded_nop argument: set it when evaluating // any script flag that is implemented as an upgraded NOP code. -void ValidateCheckInputsForAllFlags(CMutableTransaction &tx, uint32_t failing_flags, bool add_to_cache, bool upgraded_nop) +void ValidateCheckInputsForAllFlags(CMutableTransaction &tx, uint32_t failing_flags, bool add_to_cache) { PrecomputedTransactionData txdata(tx); // If we add many more flags, this loop can get too expensive, but we can @@ -119,29 +119,23 @@ void ValidateCheckInputsForAllFlags(CMutableTransaction &tx, uint32_t failing_fl // WITNESS requires P2SH test_flags |= SCRIPT_VERIFY_P2SH; } - bool ret = CheckInputs(tx, state, pcoinsTip, true, test_flags, true, add_to_cache, txdata, nullptr); + bool ret = CheckInputs(tx, state, pcoinsTip.get(), true, test_flags, true, add_to_cache, txdata, nullptr); // CheckInputs should succeed iff test_flags doesn't intersect with // failing_flags bool expected_return_value = !(test_flags & failing_flags); - if (expected_return_value && upgraded_nop) { - // If the script flag being tested corresponds to an upgraded NOP, - // then script execution should fail if DISCOURAGE_UPGRADABLE_NOPS - // is set. - expected_return_value = !(test_flags & SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS); - } BOOST_CHECK_EQUAL(ret, expected_return_value); // Test the caching if (ret && add_to_cache) { // Check that we get a cache hit if the tx was valid std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(tx, state, pcoinsTip, true, test_flags, true, add_to_cache, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputs(tx, state, pcoinsTip.get(), true, test_flags, true, add_to_cache, txdata, &scriptchecks)); BOOST_CHECK(scriptchecks.empty()); } else { // Check that we get script executions to check, if the transaction // was invalid, or we didn't add to cache. std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(tx, state, pcoinsTip, true, test_flags, true, add_to_cache, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputs(tx, state, pcoinsTip.get(), true, test_flags, true, add_to_cache, txdata, &scriptchecks)); BOOST_CHECK_EQUAL(scriptchecks.size(), tx.vin.size()); } } @@ -151,7 +145,10 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) { // Test that passing CheckInputs with one set of script flags doesn't imply // that we would pass again with a different set of flags. - InitScriptExecutionCache(); + { + LOCK(cs_main); + InitScriptExecutionCache(); + } CScript p2pk_scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; CScript p2sh_scriptPubKey = GetScriptForDestination(CScriptID(p2pk_scriptPubKey)); @@ -192,38 +189,40 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) spend_tx.vin[0].scriptSig << vchSig; } - LOCK(cs_main); - // Test that invalidity under a set of flags doesn't preclude validity // under other (eg consensus) flags. // spend_tx is invalid according to DERSIG { + LOCK(cs_main); + CValidationState state; PrecomputedTransactionData ptd_spend_tx(spend_tx); - BOOST_CHECK(!CheckInputs(spend_tx, state, pcoinsTip, true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); + BOOST_CHECK(!CheckInputs(spend_tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); // If we call again asking for scriptchecks (as happens in // ConnectBlock), we should add a script check object for this -- we're // not caching invalidity (if that changes, delete this test case). std::vector<CScriptCheck> scriptchecks; - BOOST_CHECK(CheckInputs(spend_tx, state, pcoinsTip, true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, &scriptchecks)); + BOOST_CHECK(CheckInputs(spend_tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, &scriptchecks)); BOOST_CHECK_EQUAL(scriptchecks.size(), 1); // Test that CheckInputs returns true iff DERSIG-enforcing flags are // not present. Don't add these checks to the cache, so that we can // test later that block validation works fine in the absence of cached // successes. - ValidateCheckInputsForAllFlags(spend_tx, SCRIPT_VERIFY_DERSIG | SCRIPT_VERIFY_LOW_S | SCRIPT_VERIFY_STRICTENC, false, false); + ValidateCheckInputsForAllFlags(spend_tx, SCRIPT_VERIFY_DERSIG | SCRIPT_VERIFY_LOW_S | SCRIPT_VERIFY_STRICTENC, false); + } - // And if we produce a block with this tx, it should be valid (DERSIG not - // enabled yet), even though there's no cache entry. - CBlock block; + // And if we produce a block with this tx, it should be valid (DERSIG not + // enabled yet), even though there's no cache entry. + CBlock block; - block = CreateAndProcessBlock({spend_tx}, p2pk_scriptPubKey); - BOOST_CHECK(chainActive.Tip()->GetBlockHash() == block.GetHash()); - BOOST_CHECK(pcoinsTip->GetBestBlock() == block.GetHash()); - } + block = CreateAndProcessBlock({spend_tx}, p2pk_scriptPubKey); + BOOST_CHECK(chainActive.Tip()->GetBlockHash() == block.GetHash()); + BOOST_CHECK(pcoinsTip->GetBestBlock() == block.GetHash()); + + LOCK(cs_main); // Test P2SH: construct a transaction that is valid without P2SH, and // then test validity with P2SH. @@ -239,7 +238,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) std::vector<unsigned char> vchSig2(p2pk_scriptPubKey.begin(), p2pk_scriptPubKey.end()); invalid_under_p2sh_tx.vin[0].scriptSig << vchSig2; - ValidateCheckInputsForAllFlags(invalid_under_p2sh_tx, SCRIPT_VERIFY_P2SH, true, false); + ValidateCheckInputsForAllFlags(invalid_under_p2sh_tx, SCRIPT_VERIFY_P2SH, true); } // Test CHECKLOCKTIMEVERIFY @@ -262,13 +261,13 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) vchSig.push_back((unsigned char)SIGHASH_ALL); invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 101; - ValidateCheckInputsForAllFlags(invalid_with_cltv_tx, SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true); + ValidateCheckInputsForAllFlags(invalid_with_cltv_tx, SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true); // Make it valid, and check again invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 100; CValidationState state; PrecomputedTransactionData txdata(invalid_with_cltv_tx); - BOOST_CHECK(CheckInputs(invalid_with_cltv_tx, state, pcoinsTip, true, SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); + BOOST_CHECK(CheckInputs(invalid_with_cltv_tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); } // TEST CHECKSEQUENCEVERIFY @@ -290,13 +289,13 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) vchSig.push_back((unsigned char)SIGHASH_ALL); invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 101; - ValidateCheckInputsForAllFlags(invalid_with_csv_tx, SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true); + ValidateCheckInputsForAllFlags(invalid_with_csv_tx, SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true); // Make it valid, and check again invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 100; CValidationState state; PrecomputedTransactionData txdata(invalid_with_csv_tx); - BOOST_CHECK(CheckInputs(invalid_with_csv_tx, state, pcoinsTip, true, SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); + BOOST_CHECK(CheckInputs(invalid_with_csv_tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); } // TODO: add tests for remaining script flags @@ -319,11 +318,11 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) UpdateTransaction(valid_with_witness_tx, 0, sigdata); // This should be valid under all script flags. - ValidateCheckInputsForAllFlags(valid_with_witness_tx, 0, true, false); + ValidateCheckInputsForAllFlags(valid_with_witness_tx, 0, true); // Remove the witness, and check that it is now invalid. valid_with_witness_tx.vin[0].scriptWitness.SetNull(); - ValidateCheckInputsForAllFlags(valid_with_witness_tx, SCRIPT_VERIFY_WITNESS, true, false); + ValidateCheckInputsForAllFlags(valid_with_witness_tx, SCRIPT_VERIFY_WITNESS, true); } { @@ -348,7 +347,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) } // This should be valid under all script flags - ValidateCheckInputsForAllFlags(tx, 0, true, false); + ValidateCheckInputsForAllFlags(tx, 0, true); // Check that if the second input is invalid, but the first input is // valid, the transaction is not cached. @@ -358,12 +357,12 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) CValidationState state; PrecomputedTransactionData txdata(tx); // This transaction is now invalid under segwit, because of the second input. - BOOST_CHECK(!CheckInputs(tx, state, pcoinsTip, true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); + BOOST_CHECK(!CheckInputs(tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); std::vector<CScriptCheck> scriptchecks; // Make sure this transaction was not cached (ie because the first // input was valid) - BOOST_CHECK(CheckInputs(tx, state, pcoinsTip, true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, &scriptchecks)); + BOOST_CHECK(CheckInputs(tx, state, pcoinsTip.get(), true, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, &scriptchecks)); // Should get 2 script checks back -- caching is on a whole-transaction basis. BOOST_CHECK_EQUAL(scriptchecks.size(), 2); } diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 70d83a2e54..ad5478e829 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "arith_uint256.h" -#include "uint256.h" -#include "version.h" -#include "test/test_bitcoin.h" +#include <arith_uint256.h> +#include <uint256.h> +#include <version.h> +#include <test/test_bitcoin.h> #include <boost/test/unit_test.hpp> #include <stdint.h> diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 6ec544290d..55d60d95e9 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "util.h" +#include <util.h> -#include "clientversion.h" -#include "primitives/transaction.h" -#include "sync.h" -#include "utilstrencodings.h" -#include "utilmoneystr.h" -#include "test/test_bitcoin.h" +#include <clientversion.h> +#include <primitives/transaction.h> +#include <sync.h> +#include <utilstrencodings.h> +#include <utilmoneystr.h> +#include <test/test_bitcoin.h> #include <stdint.h> #include <vector> @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(util_ParseParameters) testArgs.ParseParameters(1, (char**)argv_test); BOOST_CHECK(testArgs.GetMapArgs().empty() && testArgs.GetMapMultiArgs().empty()); - testArgs.ParseParameters(5, (char**)argv_test); + testArgs.ParseParameters(7, (char**)argv_test); // expectation: -ignored is ignored (program name argument), // -a, -b and -ccc end up in map, -d ignored because it is after // a non-option argument (non-GNU option parsing) diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 882afb2e20..12f113655a 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" -#include "versionbits.h" -#include "test/test_bitcoin.h" -#include "chainparams.h" -#include "validation.h" -#include "consensus/params.h" +#include <chain.h> +#include <versionbits.h> +#include <test/test_bitcoin.h> +#include <chainparams.h> +#include <validation.h> +#include <consensus/params.h> #include <boost/test/unit_test.hpp> @@ -32,6 +32,12 @@ public: int GetStateSinceHeightFor(const CBlockIndex* pindexPrev) const { return AbstractThresholdConditionChecker::GetStateSinceHeightFor(pindexPrev, paramsDummy, cache); } }; +class TestAlwaysActiveConditionChecker : public TestConditionChecker +{ +public: + int64_t BeginTime(const Consensus::Params& params) const override { return Consensus::BIP9Deployment::ALWAYS_ACTIVE; } +}; + #define CHECKERS 6 class VersionBitsTester @@ -43,6 +49,8 @@ class VersionBitsTester // The first one performs all checks, the second only 50%, the third only 25%, etc... // This is to test whether lack of cached information leads to the same results. TestConditionChecker checker[CHECKERS]; + // Another 6 that assume always active activation + TestAlwaysActiveConditionChecker checker_always[CHECKERS]; // Test counter (to identify failures) int num; @@ -56,6 +64,7 @@ public: } for (unsigned int i = 0; i < CHECKERS; i++) { checker[i] = TestConditionChecker(); + checker_always[i] = TestAlwaysActiveConditionChecker(); } vpblock.clear(); return *this; @@ -82,6 +91,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateSinceHeightFor(vpblock.empty() ? nullptr : vpblock.back()) == height, strprintf("Test %i for StateSinceHeight", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateSinceHeightFor(vpblock.empty() ? nullptr : vpblock.back()) == 0, strprintf("Test %i for StateSinceHeight (always active)", num)); } } num++; @@ -92,6 +102,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_DEFINED, strprintf("Test %i for DEFINED", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE (always active)", num)); } } num++; @@ -102,6 +113,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_STARTED, strprintf("Test %i for STARTED", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE (always active)", num)); } } num++; @@ -112,6 +124,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_LOCKED_IN, strprintf("Test %i for LOCKED_IN", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE (always active)", num)); } } num++; @@ -122,6 +135,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE (always active)", num)); } } num++; @@ -132,6 +146,7 @@ public: for (int i = 0; i < CHECKERS; i++) { if (InsecureRandBits(i) == 0) { BOOST_CHECK_MESSAGE(checker[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_FAILED, strprintf("Test %i for FAILED", num)); + BOOST_CHECK_MESSAGE(checker_always[i].GetStateFor(vpblock.empty() ? nullptr : vpblock.back()) == THRESHOLD_ACTIVE, strprintf("Test %i for ACTIVE (always active)", num)); } } num++; diff --git a/src/threadinterrupt.cpp b/src/threadinterrupt.cpp index 9d691079ed..5d932091cb 100644 --- a/src/threadinterrupt.cpp +++ b/src/threadinterrupt.cpp @@ -1,9 +1,9 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "threadinterrupt.h" +#include <threadinterrupt.h> CThreadInterrupt::operator bool() const { diff --git a/src/timedata.cpp b/src/timedata.cpp index 5113bb60db..a803b2fc87 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -1,19 +1,19 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "timedata.h" +#include <timedata.h> -#include "netaddress.h" -#include "sync.h" -#include "ui_interface.h" -#include "util.h" -#include "utilstrencodings.h" -#include "warnings.h" +#include <netaddress.h> +#include <sync.h> +#include <ui_interface.h> +#include <util.h> +#include <utilstrencodings.h> +#include <warnings.h> static CCriticalSection cs_nTimeOffset; diff --git a/src/timedata.h b/src/timedata.h index bc5451b19b..23584f67e5 100644 --- a/src/timedata.h +++ b/src/timedata.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/tinyformat.h b/src/tinyformat.h index 2e453e56bb..d34cfaa94f 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -495,7 +495,11 @@ namespace detail { class FormatArg { public: - FormatArg() {} + FormatArg() + : m_value(nullptr), + m_formatImpl(nullptr), + m_toIntImpl(nullptr) + { } template<typename T> explicit FormatArg(const T& value) @@ -507,11 +511,15 @@ class FormatArg void format(std::ostream& out, const char* fmtBegin, const char* fmtEnd, int ntrunc) const { + assert(m_value); + assert(m_formatImpl); m_formatImpl(out, fmtBegin, fmtEnd, ntrunc, m_value); } int toInt() const { + assert(m_value); + assert(m_toIntImpl); return m_toIntImpl(m_value); } @@ -712,23 +720,27 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositi break; case 'X': out.setf(std::ios::uppercase); + // Falls through case 'x': case 'p': out.setf(std::ios::hex, std::ios::basefield); intConversion = true; break; case 'E': out.setf(std::ios::uppercase); + // Falls through case 'e': out.setf(std::ios::scientific, std::ios::floatfield); out.setf(std::ios::dec, std::ios::basefield); break; case 'F': out.setf(std::ios::uppercase); + // Falls through case 'f': out.setf(std::ios::fixed, std::ios::floatfield); break; case 'G': out.setf(std::ios::uppercase); + // Falls through case 'g': out.setf(std::ios::dec, std::ios::basefield); // As in boost::format, let stream decide float format. diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 1cea197666..c45b5dac0d 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "torcontrol.h" -#include "utilstrencodings.h" -#include "netbase.h" -#include "net.h" -#include "util.h" -#include "crypto/hmac_sha256.h" +#include <torcontrol.h> +#include <utilstrencodings.h> +#include <netbase.h> +#include <net.h> +#include <util.h> +#include <crypto/hmac_sha256.h> #include <vector> #include <deque> diff --git a/src/torcontrol.h b/src/torcontrol.h index 72dc82c5b1..20514f7bbf 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ #ifndef BITCOIN_TORCONTROL_H #define BITCOIN_TORCONTROL_H -#include "scheduler.h" +#include <scheduler.h> extern const std::string DEFAULT_TOR_CONTROL; static const bool DEFAULT_LISTEN_ONION = true; diff --git a/src/txdb.cpp b/src/txdb.cpp index 134bb8721b..293d43c7b3 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,18 +1,18 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "txdb.h" +#include <txdb.h> -#include "chainparams.h" -#include "hash.h" -#include "random.h" -#include "pow.h" -#include "uint256.h" -#include "util.h" -#include "ui_interface.h" -#include "init.h" +#include <chainparams.h> +#include <hash.h> +#include <random.h> +#include <pow.h> +#include <uint256.h> +#include <util.h> +#include <ui_interface.h> +#include <init.h> #include <stdint.h> diff --git a/src/txdb.h b/src/txdb.h index ec9f571b13..3edeb4648e 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -1,14 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TXDB_H #define BITCOIN_TXDB_H -#include "coins.h" -#include "dbwrapper.h" -#include "chain.h" +#include <coins.h> +#include <dbwrapper.h> +#include <chain.h> #include <map> #include <string> diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 776d3f36ca..d1edde284f 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,22 +1,22 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "txmempool.h" - -#include "consensus/consensus.h" -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "validation.h" -#include "policy/policy.h" -#include "policy/fees.h" -#include "reverse_iterator.h" -#include "streams.h" -#include "timedata.h" -#include "util.h" -#include "utilmoneystr.h" -#include "utiltime.h" +#include <txmempool.h> + +#include <consensus/consensus.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <validation.h> +#include <policy/policy.h> +#include <policy/fees.h> +#include <reverse_iterator.h> +#include <streams.h> +#include <timedata.h> +#include <util.h> +#include <utilmoneystr.h> +#include <utiltime.h> CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, int64_t _nTime, unsigned int _entryHeight, @@ -317,7 +317,7 @@ void CTxMemPoolEntry::UpdateDescendantState(int64_t modifySize, CAmount modifyFe assert(int64_t(nCountWithDescendants) > 0); } -void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps) +void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps) { nSizeWithAncestors += modifySize; assert(int64_t(nSizeWithAncestors) > 0); @@ -607,6 +607,15 @@ void CTxMemPool::clear() _clear(); } +static void CheckInputsAndUpdateCoins(const CTransaction& tx, CCoinsViewCache& mempoolDuplicate, const int64_t spendheight) +{ + CValidationState state; + CAmount txfee = 0; + bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs(tx, state, mempoolDuplicate, spendheight, txfee); + assert(fCheckResult); + UpdateCoins(tx, mempoolDuplicate, 1000000); +} + void CTxMemPool::check(const CCoinsViewCache *pcoins) const { if (nCheckFrequency == 0) @@ -621,7 +630,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate(const_cast<CCoinsViewCache*>(pcoins)); - const int64_t nSpendHeight = GetSpendHeight(mempoolDuplicate); + const int64_t spendheight = GetSpendHeight(mempoolDuplicate); LOCK(cs); std::list<const CTxMemPoolEntry*> waitingOnDependants; @@ -700,11 +709,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const if (fDependsWait) waitingOnDependants.push_back(&(*it)); else { - CValidationState state; - bool fCheckResult = tx.IsCoinBase() || - Consensus::CheckTxInputs(tx, state, mempoolDuplicate, nSpendHeight); - assert(fCheckResult); - UpdateCoins(tx, mempoolDuplicate, 1000000); + CheckInputsAndUpdateCoins(tx, mempoolDuplicate, spendheight); } } unsigned int stepsSinceLastRemove = 0; @@ -717,10 +722,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const stepsSinceLastRemove++; assert(stepsSinceLastRemove < waitingOnDependants.size()); } else { - bool fCheckResult = entry->GetTx().IsCoinBase() || - Consensus::CheckTxInputs(entry->GetTx(), state, mempoolDuplicate, nSpendHeight); - assert(fCheckResult); - UpdateCoins(entry->GetTx(), mempoolDuplicate, 1000000); + CheckInputsAndUpdateCoins(entry->GetTx(), mempoolDuplicate, spendheight); stepsSinceLastRemove = 0; } } @@ -904,8 +906,8 @@ bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); - // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; + // Estimate the overhead of mapTx to be 12 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 12 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; } void CTxMemPool::RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason) { diff --git a/src/txmempool.h b/src/txmempool.h index 929d223588..ad0249c9a2 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,13 +13,13 @@ #include <utility> #include <string> -#include "amount.h" -#include "coins.h" -#include "indirectmap.h" -#include "policy/feerate.h" -#include "primitives/transaction.h" -#include "sync.h" -#include "random.h" +#include <amount.h> +#include <coins.h> +#include <indirectmap.h> +#include <policy/feerate.h> +#include <primitives/transaction.h> +#include <sync.h> +#include <random.h> #include <boost/multi_index_container.hpp> #include <boost/multi_index/hashed_index.hpp> @@ -109,7 +109,7 @@ public: // Adjusts the descendant state. void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state - void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps); + void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); // Updates the fee delta used for mining priority score, and the // modified fees with descendants. void UpdateFeeDelta(int64_t feeDelta); @@ -204,7 +204,7 @@ struct mempoolentry_txid class CompareTxMemPoolEntryByDescendantScore { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { bool fUseADescendants = UseDescendantScore(a); bool fUseBDescendants = UseDescendantScore(b); @@ -226,7 +226,7 @@ public: } // Calculate which score to use for an entry (avoiding division). - bool UseDescendantScore(const CTxMemPoolEntry &a) + bool UseDescendantScore(const CTxMemPoolEntry &a) const { double f1 = (double)a.GetModifiedFee() * a.GetSizeWithDescendants(); double f2 = (double)a.GetModFeesWithDescendants() * a.GetTxSize(); @@ -241,7 +241,7 @@ public: class CompareTxMemPoolEntryByScore { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { double f1 = (double)a.GetModifiedFee() * b.GetTxSize(); double f2 = (double)b.GetModifiedFee() * a.GetTxSize(); @@ -255,7 +255,7 @@ public: class CompareTxMemPoolEntryByEntryTime { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { return a.GetTime() < b.GetTime(); } @@ -264,7 +264,7 @@ public: class CompareTxMemPoolEntryByAncestorFee { public: - bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) const { double aFees = a.GetModFeesWithAncestors(); double aSize = a.GetSizeWithAncestors(); @@ -287,7 +287,6 @@ public: // Multi_index tag names struct descendant_score {}; struct entry_time {}; -struct mining_score {}; struct ancestor_score {}; class CBlockPolicyEstimator; @@ -356,7 +355,6 @@ public: * - transaction hash * - feerate [we use max(feerate of tx, feerate of tx with all descendants)] * - time in mempool - * - mining score (feerate modified by any fee deltas from PrioritiseTransaction) * * Note: the term "descendant" refers to in-mempool transactions that depend on * this one, while "ancestor" refers to in-mempool transactions that a given @@ -446,12 +444,6 @@ public: boost::multi_index::identity<CTxMemPoolEntry>, CompareTxMemPoolEntryByEntryTime >, - // sorted by score (for mining prioritization) - boost::multi_index::ordered_unique< - boost::multi_index::tag<mining_score>, - boost::multi_index::identity<CTxMemPoolEntry>, - CompareTxMemPoolEntryByScore - >, // sorted by fee rate with ancestors boost::multi_index::ordered_non_unique< boost::multi_index::tag<ancestor_score>, @@ -513,6 +505,9 @@ public: // to track size/count of descendant transactions. First version of // addUnchecked can be used to have it call CalculateMemPoolAncestors(), and // then invoke the second version. + // Note that addUnchecked is ONLY called from ATMP outside of tests + // and any other callers may break wallet's in-mempool tracking (due to + // lack of CValidationInterface::TransactionAddedToMempool callbacks). bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool validFeeEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool validFeeEstimate = true); diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp index 74a13e0e05..72e04f488a 100644 --- a/src/ui_interface.cpp +++ b/src/ui_interface.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2010-2016 The Bitcoin Core developers +// Copyright (c) 2010-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "ui_interface.h" -#include "util.h" +#include <ui_interface.h> +#include <util.h> CClientUIInterface uiInterface; diff --git a/src/ui_interface.h b/src/ui_interface.h index 7f68c578ee..33099b4e73 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/uint256.cpp b/src/uint256.cpp index 736a0d4fe2..82c2e01576 100644 --- a/src/uint256.cpp +++ b/src/uint256.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "uint256.h" +#include <uint256.h> -#include "utilstrencodings.h" +#include <utilstrencodings.h> #include <stdio.h> #include <string.h> diff --git a/src/uint256.h b/src/uint256.h index 94a4f7fc30..e090f10231 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,14 +12,14 @@ #include <stdint.h> #include <string> #include <vector> -#include "crypto/common.h" +#include <crypto/common.h> /** Template base class for fixed-sized opaque blobs. */ template<unsigned int BITS> class base_blob { protected: - enum { WIDTH=BITS/8 }; + static constexpr int WIDTH = BITS / 8; uint8_t data[WIDTH]; public: base_blob() diff --git a/src/undo.h b/src/undo.h index a720de4ac5..1f10c6652c 100644 --- a/src/undo.h +++ b/src/undo.h @@ -1,15 +1,15 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_UNDO_H #define BITCOIN_UNDO_H -#include "compressor.h" -#include "consensus/consensus.h" -#include "primitives/transaction.h" -#include "serialize.h" +#include <compressor.h> +#include <consensus/consensus.h> +#include <primitives/transaction.h> +#include <serialize.h> /** Undo information for a CTxIn * diff --git a/src/univalue/.travis.yml b/src/univalue/.travis.yml index 132743d349..43a1ed362e 100644 --- a/src/univalue/.travis.yml +++ b/src/univalue/.travis.yml @@ -25,7 +25,6 @@ addons: - pkg-config before_script: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew uninstall libtool; brew install libtool; fi - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh diff --git a/src/univalue/Makefile.am b/src/univalue/Makefile.am index 6c1ec81e63..e283fc890e 100644 --- a/src/univalue/Makefile.am +++ b/src/univalue/Makefile.am @@ -12,6 +12,7 @@ pkgconfig_DATA = pc/libunivalue.pc libunivalue_la_SOURCES = \ lib/univalue.cpp \ + lib/univalue_get.cpp \ lib/univalue_read.cpp \ lib/univalue_write.cpp @@ -20,7 +21,7 @@ libunivalue_la_LDFLAGS = \ -no-undefined libunivalue_la_CXXFLAGS = -I$(top_srcdir)/include -TESTS = test/unitester +TESTS = test/object test/unitester test/no_nul GENBIN = gen/gen$(BUILD_EXEEXT) GEN_SRCS = gen/gen.cpp @@ -33,7 +34,7 @@ gen: lib/univalue_escapes.h $(GENBIN) @echo Updating $< $(AM_V_at)$(GENBIN) > lib/univalue_escapes.h -noinst_PROGRAMS = $(TESTS) +noinst_PROGRAMS = $(TESTS) test/test_json TEST_DATA_DIR=test @@ -42,6 +43,21 @@ test_unitester_LDADD = libunivalue.la test_unitester_CXXFLAGS = -I$(top_srcdir)/include -DJSON_TEST_SRC=\"$(srcdir)/$(TEST_DATA_DIR)\" test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) +test_test_json_SOURCES = test/test_json.cpp +test_test_json_LDADD = libunivalue.la +test_test_json_CXXFLAGS = -I$(top_srcdir)/include +test_test_json_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +test_no_nul_SOURCES = test/no_nul.cpp +test_no_nul_LDADD = libunivalue.la +test_no_nul_CXXFLAGS = -I$(top_srcdir)/include +test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +test_object_SOURCES = test/object.cpp +test_object_LDADD = libunivalue.la +test_object_CXXFLAGS = -I$(top_srcdir)/include +test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + TEST_FILES = \ $(TEST_DATA_DIR)/fail10.json \ $(TEST_DATA_DIR)/fail11.json \ @@ -77,6 +93,8 @@ TEST_FILES = \ $(TEST_DATA_DIR)/fail39.json \ $(TEST_DATA_DIR)/fail40.json \ $(TEST_DATA_DIR)/fail41.json \ + $(TEST_DATA_DIR)/fail42.json \ + $(TEST_DATA_DIR)/fail44.json \ $(TEST_DATA_DIR)/fail3.json \ $(TEST_DATA_DIR)/fail4.json \ $(TEST_DATA_DIR)/fail5.json \ @@ -88,6 +106,11 @@ TEST_FILES = \ $(TEST_DATA_DIR)/pass2.json \ $(TEST_DATA_DIR)/pass3.json \ $(TEST_DATA_DIR)/round1.json \ - $(TEST_DATA_DIR)/round2.json + $(TEST_DATA_DIR)/round2.json \ + $(TEST_DATA_DIR)/round3.json \ + $(TEST_DATA_DIR)/round4.json \ + $(TEST_DATA_DIR)/round5.json \ + $(TEST_DATA_DIR)/round6.json \ + $(TEST_DATA_DIR)/round7.json EXTRA_DIST=$(TEST_FILES) $(GEN_SRCS) diff --git a/src/univalue/README b/src/univalue/README deleted file mode 100644 index 48167b083b..0000000000 --- a/src/univalue/README +++ /dev/null @@ -1,7 +0,0 @@ - - UniValue - -A universal value object, with JSON encoding (output) and decoding (input). - -Built as a single dynamic RAII C++ object class, and no templates. - diff --git a/src/univalue/README.md b/src/univalue/README.md new file mode 100644 index 0000000000..36aa786a4c --- /dev/null +++ b/src/univalue/README.md @@ -0,0 +1,32 @@ + +# UniValue + +## Summary + +A universal value class, with JSON encoding and decoding. + +UniValue is an abstract data type that may be a null, boolean, string, +number, array container, or a key/value dictionary container, nested to +an arbitrary depth. + +This class is aligned with the JSON standard, [RFC +7159](https://tools.ietf.org/html/rfc7159.html). + +## Installation + +This project is a standard GNU +[autotools](https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html) +project. Build and install instructions are available in the `INSTALL` +file provided with GNU autotools. + +``` +$ ./autogen.sh +$ ./configure +$ make +``` + +## Design + +UniValue provides a single dynamic RAII C++ object class, +and minimizes template use (contra json_spirit). + diff --git a/src/univalue/configure.ac b/src/univalue/configure.ac index 93d3ba945d..8298332ac1 100644 --- a/src/univalue/configure.ac +++ b/src/univalue/configure.ac @@ -1,7 +1,7 @@ m4_define([libunivalue_major_version], [1]) m4_define([libunivalue_minor_version], [1]) -m4_define([libunivalue_micro_version], [2]) -m4_define([libunivalue_interface_age], [2]) +m4_define([libunivalue_micro_version], [3]) +m4_define([libunivalue_interface_age], [3]) # If you need a modifier for the version number. # Normally empty, but can be used to make "fixup" releases. m4_define([libunivalue_extraversion], []) @@ -14,7 +14,7 @@ m4_define([libunivalue_age], [m4_eval(libunivalue_binary_age - libunivalue_inter m4_define([libunivalue_version], [libunivalue_major_version().libunivalue_minor_version().libunivalue_micro_version()libunivalue_extraversion()]) -AC_INIT([univalue], [1.0.2], +AC_INIT([univalue], [1.0.3], [http://github.com/jgarzik/univalue/]) dnl make the compilation flags quiet unless V=1 is used diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index e8ce283519..4fd2223b30 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -7,6 +7,7 @@ #define __UNIVALUE_H__ #include <stdint.h> +#include <string.h> #include <string> #include <vector> @@ -69,10 +70,11 @@ public: size_t size() const { return values.size(); } bool getBool() const { return isTrue(); } - bool checkObject(const std::map<std::string,UniValue::VType>& memberTypes); + void getObjMap(std::map<std::string,UniValue>& kv) const; + bool checkObject(const std::map<std::string,UniValue::VType>& memberTypes) const; const UniValue& operator[](const std::string& key) const; - const UniValue& operator[](unsigned int index) const; - bool exists(const std::string& key) const { return (findKey(key) >= 0); } + const UniValue& operator[](size_t index) const; + bool exists(const std::string& key) const { size_t i; return findKey(key, i); } bool isNull() const { return (typ == VNULL); } bool isTrue() const { return (typ == VBOOL) && (val == "1"); } @@ -92,8 +94,25 @@ public: std::string s(val_); return push_back(s); } + bool push_back(uint64_t val_) { + UniValue tmpVal(val_); + return push_back(tmpVal); + } + bool push_back(int64_t val_) { + UniValue tmpVal(val_); + return push_back(tmpVal); + } + bool push_back(int val_) { + UniValue tmpVal(val_); + return push_back(tmpVal); + } + bool push_back(double val_) { + UniValue tmpVal(val_); + return push_back(tmpVal); + } bool push_backV(const std::vector<UniValue>& vec); + void __pushKV(const std::string& key, const UniValue& val); bool pushKV(const std::string& key, const UniValue& val); bool pushKV(const std::string& key, const std::string& val_) { UniValue tmpVal(VSTR, val_); @@ -124,9 +143,10 @@ public: std::string write(unsigned int prettyIndent = 0, unsigned int indentLevel = 0) const; - bool read(const char *raw); + bool read(const char *raw, size_t len); + bool read(const char *raw) { return read(raw, strlen(raw)); } bool read(const std::string& rawStr) { - return read(rawStr.c_str()); + return read(rawStr.data(), rawStr.size()); } private: @@ -135,7 +155,7 @@ private: std::vector<std::string> keys; std::vector<UniValue> values; - int findKey(const std::string& key) const; + bool findKey(const std::string& key, size_t& retIdx) const; void writeArray(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const; void writeObject(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const; @@ -240,7 +260,7 @@ enum jtokentype { }; extern enum jtokentype getJsonToken(std::string& tokenVal, - unsigned int& consumed, const char *raw); + unsigned int& consumed, const char *raw, const char *end); extern const char *uvTypeName(UniValue::VType t); static inline bool jsonTokenIsValue(enum jtokentype jtt) diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 5a2860c13f..d8ad7c4b90 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -4,75 +4,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <stdint.h> -#include <errno.h> #include <iomanip> -#include <limits> #include <sstream> -#include <stdexcept> #include <stdlib.h> -#include <string.h> #include "univalue.h" -namespace -{ -static bool ParsePrechecks(const std::string& str) -{ - if (str.empty()) // No empty string allowed - return false; - if (str.size() >= 1 && (json_isspace(str[0]) || json_isspace(str[str.size()-1]))) // No padding allowed - return false; - if (str.size() != strlen(str.c_str())) // No embedded NUL characters allowed - return false; - return true; -} - -bool ParseInt32(const std::string& str, int32_t *out) -{ - if (!ParsePrechecks(str)) - return false; - char *endp = NULL; - errno = 0; // strtol will not set errno if valid - long int n = strtol(str.c_str(), &endp, 10); - if(out) *out = (int32_t)n; - // Note that strtol returns a *long int*, so even if strtol doesn't report a over/underflow - // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit - // platforms the size of these types may be different. - return endp && *endp == 0 && !errno && - n >= std::numeric_limits<int32_t>::min() && - n <= std::numeric_limits<int32_t>::max(); -} - -bool ParseInt64(const std::string& str, int64_t *out) -{ - if (!ParsePrechecks(str)) - return false; - char *endp = NULL; - errno = 0; // strtoll will not set errno if valid - long long int n = strtoll(str.c_str(), &endp, 10); - if(out) *out = (int64_t)n; - // Note that strtoll returns a *long long int*, so even if strtol doesn't report a over/underflow - // we still have to check that the returned value is within the range of an *int64_t*. - return endp && *endp == 0 && !errno && - n >= std::numeric_limits<int64_t>::min() && - n <= std::numeric_limits<int64_t>::max(); -} - -bool ParseDouble(const std::string& str, double *out) -{ - if (!ParsePrechecks(str)) - return false; - if (str.size() >= 2 && str[0] == '0' && str[1] == 'x') // No hexadecimal floats allowed - return false; - std::istringstream text(str); - text.imbue(std::locale::classic()); - double result; - text >> result; - if(out) *out = result; - return text.eof() && !text.fail(); -} -} - using namespace std; const UniValue NullUniValue; @@ -104,7 +41,7 @@ static bool validNumStr(const string& s) { string tokenVal; unsigned int consumed; - enum jtokentype tt = getJsonToken(tokenVal, consumed, s.c_str()); + enum jtokentype tt = getJsonToken(tokenVal, consumed, s.data(), s.data() + s.size()); return (tt == JTOK_NUMBER); } @@ -189,13 +126,22 @@ bool UniValue::push_backV(const std::vector<UniValue>& vec) return true; } +void UniValue::__pushKV(const std::string& key, const UniValue& val_) +{ + keys.push_back(key); + values.push_back(val_); +} + bool UniValue::pushKV(const std::string& key, const UniValue& val_) { if (typ != VOBJ) return false; - keys.push_back(key); - values.push_back(val_); + size_t idx; + if (findKey(key, idx)) + values[idx] = val_; + else + __pushKV(key, val_); return true; } @@ -204,30 +150,43 @@ bool UniValue::pushKVs(const UniValue& obj) if (typ != VOBJ || obj.typ != VOBJ) return false; - for (unsigned int i = 0; i < obj.keys.size(); i++) { - keys.push_back(obj.keys[i]); - values.push_back(obj.values.at(i)); - } + for (size_t i = 0; i < obj.keys.size(); i++) + __pushKV(obj.keys[i], obj.values.at(i)); return true; } -int UniValue::findKey(const std::string& key) const +void UniValue::getObjMap(std::map<std::string,UniValue>& kv) const +{ + if (typ != VOBJ) + return; + + kv.clear(); + for (size_t i = 0; i < keys.size(); i++) + kv[keys[i]] = values[i]; +} + +bool UniValue::findKey(const std::string& key, size_t& retIdx) const { - for (unsigned int i = 0; i < keys.size(); i++) { - if (keys[i] == key) - return (int) i; + for (size_t i = 0; i < keys.size(); i++) { + if (keys[i] == key) { + retIdx = i; + return true; + } } - return -1; + return false; } -bool UniValue::checkObject(const std::map<std::string,UniValue::VType>& t) +bool UniValue::checkObject(const std::map<std::string,UniValue::VType>& t) const { + if (typ != VOBJ) + return false; + for (std::map<std::string,UniValue::VType>::const_iterator it = t.begin(); it != t.end(); ++it) { - int idx = findKey(it->first); - if (idx < 0) + size_t idx = 0; + if (!findKey(it->first, idx)) return false; if (values.at(idx).getType() != it->second) @@ -242,14 +201,14 @@ const UniValue& UniValue::operator[](const std::string& key) const if (typ != VOBJ) return NullUniValue; - int index = findKey(key); - if (index < 0) + size_t index = 0; + if (!findKey(key, index)) return NullUniValue; return values.at(index); } -const UniValue& UniValue::operator[](unsigned int index) const +const UniValue& UniValue::operator[](size_t index) const { if (typ != VOBJ && typ != VARR) return NullUniValue; @@ -283,75 +242,3 @@ const UniValue& find_value(const UniValue& obj, const std::string& name) return NullUniValue; } -const std::vector<std::string>& UniValue::getKeys() const -{ - if (typ != VOBJ) - throw std::runtime_error("JSON value is not an object as expected"); - return keys; -} - -const std::vector<UniValue>& UniValue::getValues() const -{ - if (typ != VOBJ && typ != VARR) - throw std::runtime_error("JSON value is not an object or array as expected"); - return values; -} - -bool UniValue::get_bool() const -{ - if (typ != VBOOL) - throw std::runtime_error("JSON value is not a boolean as expected"); - return getBool(); -} - -const std::string& UniValue::get_str() const -{ - if (typ != VSTR) - throw std::runtime_error("JSON value is not a string as expected"); - return getValStr(); -} - -int UniValue::get_int() const -{ - if (typ != VNUM) - throw std::runtime_error("JSON value is not an integer as expected"); - int32_t retval; - if (!ParseInt32(getValStr(), &retval)) - throw std::runtime_error("JSON integer out of range"); - return retval; -} - -int64_t UniValue::get_int64() const -{ - if (typ != VNUM) - throw std::runtime_error("JSON value is not an integer as expected"); - int64_t retval; - if (!ParseInt64(getValStr(), &retval)) - throw std::runtime_error("JSON integer out of range"); - return retval; -} - -double UniValue::get_real() const -{ - if (typ != VNUM) - throw std::runtime_error("JSON value is not a number as expected"); - double retval; - if (!ParseDouble(getValStr(), &retval)) - throw std::runtime_error("JSON double out of range"); - return retval; -} - -const UniValue& UniValue::get_obj() const -{ - if (typ != VOBJ) - throw std::runtime_error("JSON value is not an object as expected"); - return *this; -} - -const UniValue& UniValue::get_array() const -{ - if (typ != VARR) - throw std::runtime_error("JSON value is not an array as expected"); - return *this; -} - diff --git a/src/univalue/lib/univalue_get.cpp b/src/univalue/lib/univalue_get.cpp new file mode 100644 index 0000000000..eabcf2dad1 --- /dev/null +++ b/src/univalue/lib/univalue_get.cpp @@ -0,0 +1,147 @@ +// Copyright 2014 BitPay Inc. +// Copyright 2015 Bitcoin Core Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <stdint.h> +#include <errno.h> +#include <string.h> +#include <stdlib.h> +#include <stdexcept> +#include <vector> +#include <limits> +#include <string> + +#include "univalue.h" + +namespace +{ +static bool ParsePrechecks(const std::string& str) +{ + if (str.empty()) // No empty string allowed + return false; + if (str.size() >= 1 && (json_isspace(str[0]) || json_isspace(str[str.size()-1]))) // No padding allowed + return false; + if (str.size() != strlen(str.c_str())) // No embedded NUL characters allowed + return false; + return true; +} + +bool ParseInt32(const std::string& str, int32_t *out) +{ + if (!ParsePrechecks(str)) + return false; + char *endp = NULL; + errno = 0; // strtol will not set errno if valid + long int n = strtol(str.c_str(), &endp, 10); + if(out) *out = (int32_t)n; + // Note that strtol returns a *long int*, so even if strtol doesn't report a over/underflow + // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit + // platforms the size of these types may be different. + return endp && *endp == 0 && !errno && + n >= std::numeric_limits<int32_t>::min() && + n <= std::numeric_limits<int32_t>::max(); +} + +bool ParseInt64(const std::string& str, int64_t *out) +{ + if (!ParsePrechecks(str)) + return false; + char *endp = NULL; + errno = 0; // strtoll will not set errno if valid + long long int n = strtoll(str.c_str(), &endp, 10); + if(out) *out = (int64_t)n; + // Note that strtoll returns a *long long int*, so even if strtol doesn't report a over/underflow + // we still have to check that the returned value is within the range of an *int64_t*. + return endp && *endp == 0 && !errno && + n >= std::numeric_limits<int64_t>::min() && + n <= std::numeric_limits<int64_t>::max(); +} + +bool ParseDouble(const std::string& str, double *out) +{ + if (!ParsePrechecks(str)) + return false; + if (str.size() >= 2 && str[0] == '0' && str[1] == 'x') // No hexadecimal floats allowed + return false; + std::istringstream text(str); + text.imbue(std::locale::classic()); + double result; + text >> result; + if(out) *out = result; + return text.eof() && !text.fail(); +} +} + +const std::vector<std::string>& UniValue::getKeys() const +{ + if (typ != VOBJ) + throw std::runtime_error("JSON value is not an object as expected"); + return keys; +} + +const std::vector<UniValue>& UniValue::getValues() const +{ + if (typ != VOBJ && typ != VARR) + throw std::runtime_error("JSON value is not an object or array as expected"); + return values; +} + +bool UniValue::get_bool() const +{ + if (typ != VBOOL) + throw std::runtime_error("JSON value is not a boolean as expected"); + return getBool(); +} + +const std::string& UniValue::get_str() const +{ + if (typ != VSTR) + throw std::runtime_error("JSON value is not a string as expected"); + return getValStr(); +} + +int UniValue::get_int() const +{ + if (typ != VNUM) + throw std::runtime_error("JSON value is not an integer as expected"); + int32_t retval; + if (!ParseInt32(getValStr(), &retval)) + throw std::runtime_error("JSON integer out of range"); + return retval; +} + +int64_t UniValue::get_int64() const +{ + if (typ != VNUM) + throw std::runtime_error("JSON value is not an integer as expected"); + int64_t retval; + if (!ParseInt64(getValStr(), &retval)) + throw std::runtime_error("JSON integer out of range"); + return retval; +} + +double UniValue::get_real() const +{ + if (typ != VNUM) + throw std::runtime_error("JSON value is not a number as expected"); + double retval; + if (!ParseDouble(getValStr(), &retval)) + throw std::runtime_error("JSON double out of range"); + return retval; +} + +const UniValue& UniValue::get_obj() const +{ + if (typ != VOBJ) + throw std::runtime_error("JSON value is not an object as expected"); + return *this; +} + +const UniValue& UniValue::get_array() const +{ + if (typ != VARR) + throw std::runtime_error("JSON value is not an array as expected"); + return *this; +} + diff --git a/src/univalue/lib/univalue_read.cpp b/src/univalue/lib/univalue_read.cpp index 95bac6958d..ae75cb462a 100644 --- a/src/univalue/lib/univalue_read.cpp +++ b/src/univalue/lib/univalue_read.cpp @@ -43,21 +43,21 @@ static const char *hatoui(const char *first, const char *last, } enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, - const char *raw) + const char *raw, const char *end) { tokenVal.clear(); consumed = 0; const char *rawStart = raw; - while ((*raw) && (json_isspace(*raw))) // skip whitespace + while (raw < end && (json_isspace(*raw))) // skip whitespace raw++; - switch (*raw) { - - case 0: + if (raw >= end) return JTOK_NONE; + switch (*raw) { + case '{': raw++; consumed = (raw - rawStart); @@ -127,40 +127,40 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, numStr += *raw; // copy first char raw++; - if ((*first == '-') && (!json_isdigit(*raw))) + if ((*first == '-') && (raw < end) && (!json_isdigit(*raw))) return JTOK_ERR; - while ((*raw) && json_isdigit(*raw)) { // copy digits + while (raw < end && json_isdigit(*raw)) { // copy digits numStr += *raw; raw++; } // part 2: frac - if (*raw == '.') { + if (raw < end && *raw == '.') { numStr += *raw; // copy . raw++; - if (!json_isdigit(*raw)) + if (raw >= end || !json_isdigit(*raw)) return JTOK_ERR; - while ((*raw) && json_isdigit(*raw)) { // copy digits + while (raw < end && json_isdigit(*raw)) { // copy digits numStr += *raw; raw++; } } // part 3: exp - if (*raw == 'e' || *raw == 'E') { + if (raw < end && (*raw == 'e' || *raw == 'E')) { numStr += *raw; // copy E raw++; - if (*raw == '-' || *raw == '+') { // copy +/- + if (raw < end && (*raw == '-' || *raw == '+')) { // copy +/- numStr += *raw; raw++; } - if (!json_isdigit(*raw)) + if (raw >= end || !json_isdigit(*raw)) return JTOK_ERR; - while ((*raw) && json_isdigit(*raw)) { // copy digits + while (raw < end && json_isdigit(*raw)) { // copy digits numStr += *raw; raw++; } @@ -177,13 +177,16 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, string valStr; JSONUTF8StringFilter writer(valStr); - while (*raw) { - if ((unsigned char)*raw < 0x20) + while (true) { + if (raw >= end || (unsigned char)*raw < 0x20) return JTOK_ERR; else if (*raw == '\\') { raw++; // skip backslash + if (raw >= end) + return JTOK_ERR; + switch (*raw) { case '"': writer.push_back('\"'); break; case '\\': writer.push_back('\\'); break; @@ -196,7 +199,8 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, case 'u': { unsigned int codepoint; - if (hatoui(raw + 1, raw + 1 + 4, codepoint) != + if (raw + 1 + 4 >= end || + hatoui(raw + 1, raw + 1 + 4, codepoint) != raw + 1 + 4) return JTOK_ERR; writer.push_back_u(codepoint); @@ -246,7 +250,7 @@ enum expect_bits { #define setExpect(bit) (expectMask |= EXP_##bit) #define clearExpect(bit) (expectMask &= ~EXP_##bit) -bool UniValue::read(const char *raw) +bool UniValue::read(const char *raw, size_t size) { clear(); @@ -257,10 +261,11 @@ bool UniValue::read(const char *raw) unsigned int consumed; enum jtokentype tok = JTOK_NONE; enum jtokentype last_tok = JTOK_NONE; + const char* end = raw + size; do { last_tok = tok; - tok = getJsonToken(tokenVal, consumed, raw); + tok = getJsonToken(tokenVal, consumed, raw, end); if (tok == JTOK_NONE || tok == JTOK_ERR) return false; raw += consumed; @@ -371,9 +376,6 @@ bool UniValue::read(const char *raw) case JTOK_KW_NULL: case JTOK_KW_TRUE: case JTOK_KW_FALSE: { - if (!stack.size()) - return false; - UniValue tmpVal; switch (tok) { case JTOK_KW_NULL: @@ -388,6 +390,11 @@ bool UniValue::read(const char *raw) default: /* impossible */ break; } + if (!stack.size()) { + *this = tmpVal; + break; + } + UniValue *top = stack.back(); top->values.push_back(tmpVal); @@ -396,10 +403,12 @@ bool UniValue::read(const char *raw) } case JTOK_NUMBER: { - if (!stack.size()) - return false; - UniValue tmpVal(VNUM, tokenVal); + if (!stack.size()) { + *this = tmpVal; + break; + } + UniValue *top = stack.back(); top->values.push_back(tmpVal); @@ -408,17 +417,18 @@ bool UniValue::read(const char *raw) } case JTOK_STRING: { - if (!stack.size()) - return false; - - UniValue *top = stack.back(); - if (expect(OBJ_NAME)) { + UniValue *top = stack.back(); top->keys.push_back(tokenVal); clearExpect(OBJ_NAME); setExpect(COLON); } else { UniValue tmpVal(VSTR, tokenVal); + if (!stack.size()) { + *this = tmpVal; + break; + } + UniValue *top = stack.back(); top->values.push_back(tmpVal); } @@ -432,7 +442,7 @@ bool UniValue::read(const char *raw) } while (!stack.empty ()); /* Check that nothing follows the initial construct (parsed above). */ - tok = getJsonToken(tokenVal, consumed, raw); + tok = getJsonToken(tokenVal, consumed, raw, end); if (tok != JTOK_NONE) return false; diff --git a/src/univalue/lib/univalue_utffilter.h b/src/univalue/lib/univalue_utffilter.h index 2fb6a492d1..20d4043009 100644 --- a/src/univalue/lib/univalue_utffilter.h +++ b/src/univalue/lib/univalue_utffilter.h @@ -46,19 +46,19 @@ public: } } // Write codepoint directly, possibly collating surrogate pairs - void push_back_u(unsigned int codepoint) + void push_back_u(unsigned int codepoint_) { if (state) // Only accept full codepoints in open state is_valid = false; - if (codepoint >= 0xD800 && codepoint < 0xDC00) { // First half of surrogate pair + if (codepoint_ >= 0xD800 && codepoint_ < 0xDC00) { // First half of surrogate pair if (surpair) // Two subsequent surrogate pair openers - fail is_valid = false; else - surpair = codepoint; - } else if (codepoint >= 0xDC00 && codepoint < 0xE000) { // Second half of surrogate pair + surpair = codepoint_; + } else if (codepoint_ >= 0xDC00 && codepoint_ < 0xE000) { // Second half of surrogate pair if (surpair) { // Open surrogate pair, expect second half // Compute code point from UTF-16 surrogate pair - append_codepoint(0x10000 | ((surpair - 0xD800)<<10) | (codepoint - 0xDC00)); + append_codepoint(0x10000 | ((surpair - 0xD800)<<10) | (codepoint_ - 0xDC00)); surpair = 0; } else // Second half doesn't follow a first half - fail is_valid = false; @@ -66,7 +66,7 @@ public: if (surpair) // First half of surrogate pair not followed by second - fail is_valid = false; else - append_codepoint(codepoint); + append_codepoint(codepoint_); } } // Check that we're in a state where the string can be ended @@ -96,22 +96,22 @@ private: // Two subsequent \u.... may have to be replaced with one actual codepoint. unsigned int surpair; // First half of open UTF-16 surrogate pair, or 0 - void append_codepoint(unsigned int codepoint) + void append_codepoint(unsigned int codepoint_) { - if (codepoint <= 0x7f) - str.push_back((char)codepoint); - else if (codepoint <= 0x7FF) { - str.push_back((char)(0xC0 | (codepoint >> 6))); - str.push_back((char)(0x80 | (codepoint & 0x3F))); - } else if (codepoint <= 0xFFFF) { - str.push_back((char)(0xE0 | (codepoint >> 12))); - str.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F))); - str.push_back((char)(0x80 | (codepoint & 0x3F))); - } else if (codepoint <= 0x1FFFFF) { - str.push_back((char)(0xF0 | (codepoint >> 18))); - str.push_back((char)(0x80 | ((codepoint >> 12) & 0x3F))); - str.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F))); - str.push_back((char)(0x80 | (codepoint & 0x3F))); + if (codepoint_ <= 0x7f) + str.push_back((char)codepoint_); + else if (codepoint_ <= 0x7FF) { + str.push_back((char)(0xC0 | (codepoint_ >> 6))); + str.push_back((char)(0x80 | (codepoint_ & 0x3F))); + } else if (codepoint_ <= 0xFFFF) { + str.push_back((char)(0xE0 | (codepoint_ >> 12))); + str.push_back((char)(0x80 | ((codepoint_ >> 6) & 0x3F))); + str.push_back((char)(0x80 | (codepoint_ & 0x3F))); + } else if (codepoint_ <= 0x1FFFFF) { + str.push_back((char)(0xF0 | (codepoint_ >> 18))); + str.push_back((char)(0x80 | ((codepoint_ >> 12) & 0x3F))); + str.push_back((char)(0x80 | ((codepoint_ >> 6) & 0x3F))); + str.push_back((char)(0x80 | (codepoint_ & 0x3F))); } } }; diff --git a/src/univalue/lib/univalue_write.cpp b/src/univalue/lib/univalue_write.cpp index cfbdad3284..cf27835991 100644 --- a/src/univalue/lib/univalue_write.cpp +++ b/src/univalue/lib/univalue_write.cpp @@ -79,8 +79,6 @@ void UniValue::writeArray(unsigned int prettyIndent, unsigned int indentLevel, s s += values[i].write(prettyIndent, indentLevel + 1); if (i != (values.size() - 1)) { s += ","; - if (prettyIndent) - s += " "; } if (prettyIndent) s += "\n"; diff --git a/src/univalue/test/.gitignore b/src/univalue/test/.gitignore index 3d9347fe7e..7b27cf0da2 100644 --- a/src/univalue/test/.gitignore +++ b/src/univalue/test/.gitignore @@ -1,4 +1,8 @@ + +object unitester +test_json +no_nul *.trs *.log diff --git a/src/univalue/test/fail1.json b/src/univalue/test/fail1.json index 6216b865f1..8feb01a6d0 100644 --- a/src/univalue/test/fail1.json +++ b/src/univalue/test/fail1.json @@ -1 +1 @@ -"A JSON payload should be an object or array, not a string."
\ No newline at end of file +"This is a string that never ends, yes it goes on and on, my friends. diff --git a/src/univalue/test/fail42.json b/src/univalue/test/fail42.json Binary files differnew file mode 100644 index 0000000000..9c7565adbd --- /dev/null +++ b/src/univalue/test/fail42.json diff --git a/src/univalue/test/fail44.json b/src/univalue/test/fail44.json new file mode 100644 index 0000000000..80edceddf1 --- /dev/null +++ b/src/univalue/test/fail44.json @@ -0,0 +1 @@ +"This file ends without a newline or close-quote.
\ No newline at end of file diff --git a/src/univalue/test/no_nul.cpp b/src/univalue/test/no_nul.cpp new file mode 100644 index 0000000000..83d292200b --- /dev/null +++ b/src/univalue/test/no_nul.cpp @@ -0,0 +1,8 @@ +#include "univalue.h" + +int main (int argc, char *argv[]) +{ + char buf[] = "___[1,2,3]___"; + UniValue val; + return val.read(buf + 3, 7) ? 0 : 1; +} diff --git a/src/test/univalue_tests.cpp b/src/univalue/test/object.cpp index dffe8e55a8..679cc9f143 100644 --- a/src/test/univalue_tests.cpp +++ b/src/univalue/test/object.cpp @@ -7,10 +7,31 @@ #include <vector> #include <string> #include <map> +#include <cassert> +#include <stdexcept> #include <univalue.h> -#include "test/test_bitcoin.h" -#include <boost/test/unit_test.hpp> +#define BOOST_FIXTURE_TEST_SUITE(a, b) +#define BOOST_AUTO_TEST_CASE(funcName) void funcName() +#define BOOST_AUTO_TEST_SUITE_END() +#define BOOST_CHECK(expr) assert(expr) +#define BOOST_CHECK_EQUAL(v1, v2) assert((v1) == (v2)) +#define BOOST_CHECK_THROW(stmt, excMatch) { \ + try { \ + (stmt); \ + assert(0 && "No exception caught"); \ + } catch (excMatch & e) { \ + } catch (...) { \ + assert(0 && "Wrong exception caught"); \ + } \ + } +#define BOOST_CHECK_NO_THROW(stmt) { \ + try { \ + (stmt); \ + } catch (...) { \ + assert(0); \ + } \ + } BOOST_FIXTURE_TEST_SUITE(univalue_tests, BasicTestingSetup) @@ -185,14 +206,23 @@ BOOST_AUTO_TEST_CASE(univalue_array) BOOST_CHECK(arr.push_backV(vec)); + BOOST_CHECK(arr.push_back((uint64_t) 400ULL)); + BOOST_CHECK(arr.push_back((int64_t) -400LL)); + BOOST_CHECK(arr.push_back((int) -401)); + BOOST_CHECK(arr.push_back(-40.1)); + BOOST_CHECK_EQUAL(arr.empty(), false); - BOOST_CHECK_EQUAL(arr.size(), 5); + BOOST_CHECK_EQUAL(arr.size(), 9); BOOST_CHECK_EQUAL(arr[0].getValStr(), "1023"); BOOST_CHECK_EQUAL(arr[1].getValStr(), "zippy"); BOOST_CHECK_EQUAL(arr[2].getValStr(), "pippy"); BOOST_CHECK_EQUAL(arr[3].getValStr(), "boing"); BOOST_CHECK_EQUAL(arr[4].getValStr(), "going"); + BOOST_CHECK_EQUAL(arr[5].getValStr(), "400"); + BOOST_CHECK_EQUAL(arr[6].getValStr(), "-400"); + BOOST_CHECK_EQUAL(arr[7].getValStr(), "-401"); + BOOST_CHECK_EQUAL(arr[8].getValStr(), "-40.1"); BOOST_CHECK_EQUAL(arr[999].getValStr(), ""); @@ -282,6 +312,27 @@ BOOST_AUTO_TEST_CASE(univalue_object) obj.clear(); BOOST_CHECK(obj.empty()); BOOST_CHECK_EQUAL(obj.size(), 0); + BOOST_CHECK_EQUAL(obj.getType(), UniValue::VNULL); + + BOOST_CHECK_EQUAL(obj.setObject(), true); + UniValue uv; + uv.setInt(42); + obj.__pushKV("age", uv); + BOOST_CHECK_EQUAL(obj.size(), 1); + BOOST_CHECK_EQUAL(obj["age"].getValStr(), "42"); + + uv.setInt(43); + obj.pushKV("age", uv); + BOOST_CHECK_EQUAL(obj.size(), 1); + BOOST_CHECK_EQUAL(obj["age"].getValStr(), "43"); + + obj.pushKV("name", "foo bar"); + + std::map<std::string,UniValue> kv; + obj.getObjMap(kv); + BOOST_CHECK_EQUAL(kv["age"].getValStr(), "43"); + BOOST_CHECK_EQUAL(kv["name"].getValStr(), "foo bar"); + } static const char *json1 = @@ -331,3 +382,15 @@ BOOST_AUTO_TEST_CASE(univalue_readwrite) } BOOST_AUTO_TEST_SUITE_END() + +int main (int argc, char *argv[]) +{ + univalue_constructor(); + univalue_typecheck(); + univalue_set(); + univalue_array(); + univalue_object(); + univalue_readwrite(); + return 0; +} + diff --git a/src/univalue/test/round3.json b/src/univalue/test/round3.json new file mode 100644 index 0000000000..7182dc2f9b --- /dev/null +++ b/src/univalue/test/round3.json @@ -0,0 +1 @@ +"abcdefghijklmnopqrstuvwxyz" diff --git a/src/univalue/test/round4.json b/src/univalue/test/round4.json new file mode 100644 index 0000000000..7f8f011eb7 --- /dev/null +++ b/src/univalue/test/round4.json @@ -0,0 +1 @@ +7 diff --git a/src/univalue/test/round5.json b/src/univalue/test/round5.json new file mode 100644 index 0000000000..27ba77ddaf --- /dev/null +++ b/src/univalue/test/round5.json @@ -0,0 +1 @@ +true diff --git a/src/univalue/test/round6.json b/src/univalue/test/round6.json new file mode 100644 index 0000000000..c508d5366f --- /dev/null +++ b/src/univalue/test/round6.json @@ -0,0 +1 @@ +false diff --git a/src/univalue/test/round7.json b/src/univalue/test/round7.json new file mode 100644 index 0000000000..19765bd501 --- /dev/null +++ b/src/univalue/test/round7.json @@ -0,0 +1 @@ +null diff --git a/src/univalue/test/test_json.cpp b/src/univalue/test/test_json.cpp new file mode 100644 index 0000000000..2943bae2b1 --- /dev/null +++ b/src/univalue/test/test_json.cpp @@ -0,0 +1,24 @@ +// Test program that can be called by the JSON test suite at +// https://github.com/nst/JSONTestSuite. +// +// It reads JSON input from stdin and exits with code 0 if it can be parsed +// successfully. It also pretty prints the parsed JSON value to stdout. + +#include <iostream> +#include <string> +#include "univalue.h" + +using namespace std; + +int main (int argc, char *argv[]) +{ + UniValue val; + if (val.read(string(istreambuf_iterator<char>(cin), + istreambuf_iterator<char>()))) { + cout << val.write(1 /* prettyIndent */, 4 /* indentLevel */) << endl; + return 0; + } else { + cerr << "JSON Parse Error." << endl; + return 1; + } +} diff --git a/src/univalue/test/unitester.cpp b/src/univalue/test/unitester.cpp index 05f3842cd1..2c37794a4b 100644 --- a/src/univalue/test/unitester.cpp +++ b/src/univalue/test/unitester.cpp @@ -113,6 +113,8 @@ static const char *filenames[] = { "fail39.json", // invalid unicode: only second half of surrogate pair "fail40.json", // invalid unicode: broken UTF-8 "fail41.json", // invalid unicode: unfinished UTF-8 + "fail42.json", // valid json with garbage following a nul byte + "fail44.json", // unterminated string "fail3.json", "fail4.json", // extra comma "fail5.json", @@ -125,6 +127,11 @@ static const char *filenames[] = { "pass3.json", "round1.json", // round-trip test "round2.json", // unicode + "round3.json", // bare string + "round4.json", // bare number + "round5.json", // bare true + "round6.json", // bare false + "round7.json", // bare null }; // Test \u handling diff --git a/src/util.cpp b/src/util.cpp index 51ccc94787..150bc503df 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -1,20 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" -#endif - -#include "util.h" +#include <util.h> -#include "chainparamsbase.h" -#include "fs.h" -#include "random.h" -#include "serialize.h" -#include "utilstrencodings.h" -#include "utiltime.h" +#include <chainparamsbase.h> +#include <random.h> +#include <serialize.h> +#include <utilstrencodings.h> #include <stdarg.h> @@ -89,6 +83,7 @@ const int64_t nStartupTime = GetTime(); const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char * const BITCOIN_PID_FILENAME = "bitcoind.pid"; +const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; ArgsManager gArgs; bool fPrintToConsole = false; @@ -189,26 +184,40 @@ static void DebugPrintInit() vMsgsBeforeOpenLog = new std::list<std::string>; } -void OpenDebugLog() +fs::path GetDebugLogPath() +{ + fs::path logfile(gArgs.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); + if (logfile.is_absolute()) { + return logfile; + } else { + return GetDataDir() / logfile; + } +} + +bool OpenDebugLog() { boost::call_once(&DebugPrintInit, debugPrintInitFlag); boost::mutex::scoped_lock scoped_lock(*mutexDebugLog); assert(fileout == nullptr); assert(vMsgsBeforeOpenLog); - fs::path pathDebug = GetDataDir() / "debug.log"; + fs::path pathDebug = GetDebugLogPath(); + fileout = fsbridge::fopen(pathDebug, "a"); - if (fileout) { - setbuf(fileout, nullptr); // unbuffered - // dump buffered messages from before we opened the log - while (!vMsgsBeforeOpenLog->empty()) { - FileWriteStr(vMsgsBeforeOpenLog->front(), fileout); - vMsgsBeforeOpenLog->pop_front(); - } + if (!fileout) { + return false; + } + + setbuf(fileout, nullptr); // unbuffered + // dump buffered messages from before we opened the log + while (!vMsgsBeforeOpenLog->empty()) { + FileWriteStr(vMsgsBeforeOpenLog->front(), fileout); + vMsgsBeforeOpenLog->pop_front(); } delete vMsgsBeforeOpenLog; vMsgsBeforeOpenLog = nullptr; + return true; } struct CLogCategoryDesc @@ -220,6 +229,7 @@ struct CLogCategoryDesc const CLogCategoryDesc LogCategories[] = { {BCLog::NONE, "0"}, + {BCLog::NONE, "none"}, {BCLog::NET, "net"}, {BCLog::TOR, "tor"}, {BCLog::MEMPOOL, "mempool"}, @@ -354,7 +364,7 @@ int LogPrintStr(const std::string &str) // reopen the log file, if requested if (fReopenDebugLog) { fReopenDebugLog = false; - fs::path pathDebug = GetDataDir() / "debug.log"; + fs::path pathDebug = GetDebugLogPath(); if (fsbridge::freopen(pathDebug,"a",fileout) != nullptr) setbuf(fileout, nullptr); // unbuffered } @@ -574,7 +584,10 @@ const fs::path &GetDataDir(bool fNetSpecific) if (fNetSpecific) path /= BaseParams().DataDir(); - fs::create_directories(path); + if (fs::create_directories(path)) { + // This is the first run, create wallets subdirectory too + fs::create_directories(path / "wallets"); + } return path; } @@ -620,6 +633,9 @@ void ArgsManager::ReadConfigFile(const std::string& confPath) } // If datadir is changed in .conf file: ClearDatadirCache(); + if (!fs::is_directory(GetDataDir(false))) { + throw std::runtime_error(strprintf("specified data directory \"%s\" does not exist.", gArgs.GetArg("-datadir", "").c_str())); + } } #ifndef WIN32 @@ -770,7 +786,7 @@ void ShrinkDebugFile() // Amount of debug.log to save at end when shrinking (must fit in memory) constexpr size_t RECENT_DEBUG_HISTORY_SIZE = 10 * 1000000; // Scroll debug.log if it's getting too big - fs::path pathLog = GetDataDir() / "debug.log"; + fs::path pathLog = GetDebugLogPath(); FILE* file = fsbridge::fopen(pathLog, "r"); // If debug.log file is more than 10% bigger the RECENT_DEBUG_HISTORY_SIZE // trim it down by saving only the last RECENT_DEBUG_HISTORY_SIZE bytes diff --git a/src/util.h b/src/util.h index 480a80c0a3..6a0d6a31e7 100644 --- a/src/util.h +++ b/src/util.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,14 +11,14 @@ #define BITCOIN_UTIL_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "compat.h" -#include "fs.h" -#include "sync.h" -#include "tinyformat.h" -#include "utiltime.h" +#include <compat.h> +#include <fs.h> +#include <sync.h> +#include <tinyformat.h> +#include <utiltime.h> #include <atomic> #include <exception> @@ -28,6 +28,7 @@ #include <vector> #include <boost/signals2/signal.hpp> +#include <boost/thread/condition_variable.hpp> // for boost::thread_interrupted // Application startup time (used for uptime calculation) int64_t GetStartupTime(); @@ -35,6 +36,7 @@ int64_t GetStartupTime(); static const bool DEFAULT_LOGTIMEMICROS = false; static const bool DEFAULT_LOGIPS = false; static const bool DEFAULT_LOGTIMESTAMPS = true; +extern const char * const DEFAULT_DEBUGLOGFILE; /** Signals for translation. */ class CTranslationInterface @@ -132,6 +134,10 @@ template<typename T, typename... Args> static inline void MarkUsed(const T& t, c MarkUsed(args...); } +// Be conservative when using LogPrintf/error or other things which +// unconditionally log to debug.log! It should not be the case that an inbound +// peer can fill up a users disk with debug.log entries. + #ifdef USE_COVERAGE #define LogPrintf(...) do { MarkUsed(__VA_ARGS__); } while(0) #define LogPrint(category, ...) do { MarkUsed(__VA_ARGS__); } while(0) @@ -179,7 +185,8 @@ void CreatePidFile(const fs::path &path, pid_t pid); #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif -void OpenDebugLog(); +fs::path GetDebugLogPath(); +bool OpenDebugLog(); void ShrinkDebugFile(); void runCommand(const std::string& strCommand); @@ -326,4 +333,11 @@ template <typename Callable> void TraceThread(const char* name, Callable func) std::string CopyrightHolders(const std::string& strPrefix); +//! Substitute for C++14 std::make_unique. +template <typename T, typename... Args> +std::unique_ptr<T> MakeUnique(Args&&... args) +{ + return std::unique_ptr<T>(new T(std::forward<Args>(args)...)); +} + #endif // BITCOIN_UTIL_H diff --git a/src/utilmoneystr.cpp b/src/utilmoneystr.cpp index 6e6e33184e..ea8a817ac4 100644 --- a/src/utilmoneystr.cpp +++ b/src/utilmoneystr.cpp @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "utilmoneystr.h" +#include <utilmoneystr.h> -#include "primitives/transaction.h" -#include "tinyformat.h" -#include "utilstrencodings.h" +#include <primitives/transaction.h> +#include <tinyformat.h> +#include <utilstrencodings.h> std::string FormatMoney(const CAmount& n) { diff --git a/src/utilmoneystr.h b/src/utilmoneystr.h index bc885ee167..1cbec49858 100644 --- a/src/utilmoneystr.h +++ b/src/utilmoneystr.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ #include <stdint.h> #include <string> -#include "amount.h" +#include <amount.h> /* Do not use these functions to represent or parse monetary amounts to or from * JSON but use AmountFromValue and ValueFromAmount for that. diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 741680e93f..52158e9804 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "utilstrencodings.h" +#include <utilstrencodings.h> -#include "tinyformat.h" +#include <tinyformat.h> #include <cstdlib> #include <cstring> diff --git a/src/utilstrencodings.h b/src/utilstrencodings.h index af33f0e5f8..994e6abbad 100644 --- a/src/utilstrencodings.h +++ b/src/utilstrencodings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/utiltime.cpp b/src/utiltime.cpp index 4cc77dbfeb..e908173135 100644 --- a/src/utiltime.cpp +++ b/src/utiltime.cpp @@ -1,13 +1,13 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "utiltime.h" +#include <utiltime.h> #include <atomic> diff --git a/src/utiltime.h b/src/utiltime.h index 8ae8540b89..56cc31da67 100644 --- a/src/utiltime.h +++ b/src/utiltime.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2015 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/validation.cpp b/src/validation.cpp index e098de5d3d..14d60bb269 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1,48 +1,46 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "validation.h" - -#include "arith_uint256.h" -#include "chain.h" -#include "chainparams.h" -#include "checkpoints.h" -#include "checkqueue.h" -#include "consensus/consensus.h" -#include "consensus/merkle.h" -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "cuckoocache.h" -#include "fs.h" -#include "hash.h" -#include "init.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "pow.h" -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "random.h" -#include "reverse_iterator.h" -#include "script/script.h" -#include "script/sigcache.h" -#include "script/standard.h" -#include "timedata.h" -#include "tinyformat.h" -#include "txdb.h" -#include "txmempool.h" -#include "ui_interface.h" -#include "undo.h" -#include "util.h" -#include "utilmoneystr.h" -#include "utilstrencodings.h" -#include "validationinterface.h" -#include "versionbits.h" -#include "warnings.h" - -#include <atomic> +#include <validation.h> + +#include <arith_uint256.h> +#include <chain.h> +#include <chainparams.h> +#include <checkpoints.h> +#include <checkqueue.h> +#include <consensus/consensus.h> +#include <consensus/merkle.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <cuckoocache.h> +#include <hash.h> +#include <init.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <pow.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <random.h> +#include <reverse_iterator.h> +#include <script/script.h> +#include <script/sigcache.h> +#include <script/standard.h> +#include <timedata.h> +#include <tinyformat.h> +#include <txdb.h> +#include <txmempool.h> +#include <ui_interface.h> +#include <undo.h> +#include <util.h> +#include <utilmoneystr.h> +#include <utilstrencodings.h> +#include <validationinterface.h> +#include <warnings.h> + +#include <future> #include <sstream> #include <boost/algorithm/string/replace.hpp> @@ -59,11 +57,150 @@ /** * Global state */ +namespace { + struct CBlockIndexWorkComparator + { + bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { + // First sort by most total work, ... + if (pa->nChainWork > pb->nChainWork) return false; + if (pa->nChainWork < pb->nChainWork) return true; + + // ... then by earliest time received, ... + if (pa->nSequenceId < pb->nSequenceId) return false; + if (pa->nSequenceId > pb->nSequenceId) return true; + + // Use pointer address as tie breaker (should only happen with blocks + // loaded from disk, as those all have id 0). + if (pa < pb) return false; + if (pa > pb) return true; + + // Identical blocks. + return false; + } + }; +} // anon namespace + +enum DisconnectResult +{ + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; + +class ConnectTrace; + +/** + * CChainState stores and provides an API to update our local knowledge of the + * current best chain and header tree. + * + * It generally provides access to the current block tree, as well as functions + * to provide new data, which it will appropriately validate and incorporate in + * its state as necessary. + * + * Eventually, the API here is targeted at being exposed externally as a + * consumable libconsensus library, so any functions added must only call + * other class member functions, pure functions in other parts of the consensus + * library, callbacks via the validation interface, or read/write-to-disk + * functions (eventually this will also be via callbacks). + */ +class CChainState { +private: + /** + * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and + * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be + * missing the data for the block. + */ + std::set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; + + /** + * Every received block is assigned a unique and increasing identifier, so we + * know which one to give priority in case of a fork. + */ + CCriticalSection cs_nBlockSequenceId; + /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ + int32_t nBlockSequenceId = 1; + /** Decreasing counter (used by subsequent preciousblock calls). */ + int32_t nBlockReverseSequenceId = -1; + /** chainwork for the last block that preciousblock has been applied to. */ + arith_uint256 nLastPreciousChainwork = 0; + + /** In order to efficiently track invalidity of headers, we keep the set of + * blocks which we tried to connect and found to be invalid here (ie which + * were set to BLOCK_FAILED_VALID since the last restart). We can then + * walk this set and check if a new header is a descendant of something in + * this set, preventing us from having to walk mapBlockIndex when we try + * to connect a bad block and fail. + * + * While this is more complicated than marking everything which descends + * from an invalid block as invalid at the time we discover it to be + * invalid, doing so would require walking all of mapBlockIndex to find all + * descendants. Since this case should be very rare, keeping track of all + * BLOCK_FAILED_VALID blocks in a set should be just fine and work just as + * well. + * + * Because we already walk mapBlockIndex in height-order at startup, we go + * ahead and mark descendants of invalid blocks as FAILED_CHILD at that time, + * instead of putting things in this set. + */ + std::set<CBlockIndex*> g_failed_blocks; + +public: + CChain chainActive; + BlockMap mapBlockIndex; + std::multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; + CBlockIndex *pindexBestInvalid = nullptr; + + bool LoadBlockIndex(const Consensus::Params& consensus_params, CBlockTreeDB& blocktree); + + bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock); + + bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex); + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock); + + // Block (dis)connection on a given view: + DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); + bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false); + + // Block disconnection on our pcoinsTip: + bool DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool); + + // Manual block validity manipulation: + bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex); + bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex); + bool ResetBlockFailureFlags(CBlockIndex *pindex); + + bool ReplayBlocks(const CChainParams& params, CCoinsView* view); + bool RewindBlockIndex(const CChainParams& params); + bool LoadGenesisBlock(const CChainParams& chainparams); + + void PruneBlockIndexCandidates(); + + void UnloadBlockIndex(); + +private: + bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace); + bool ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool); + + CBlockIndex* AddToBlockIndex(const CBlockHeader& block); + /** Create a new block index entry for a given block hash */ + CBlockIndex * InsertBlockIndex(const uint256& hash); + void CheckBlockIndex(const Consensus::Params& consensusParams); + + void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state); + CBlockIndex* FindMostWorkChain(); + bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams); + + + bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params); +} g_chainstate; + + CCriticalSection cs_main; -BlockMap mapBlockIndex; -CChain chainActive; +BlockMap& mapBlockIndex = g_chainstate.mapBlockIndex; +CChain& chainActive = g_chainstate.chainActive; CBlockIndex *pindexBestHeader = nullptr; CWaitableCriticalSection csBestBlock; CConditionVariable cvBlockChange; @@ -91,8 +228,6 @@ CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CBlockPolicyEstimator feeEstimator; CTxMemPool mempool(&feeEstimator); -static void CheckBlockIndex(const Consensus::Params& consensusParams); - /** Constant stuff for coinbase transactions we create: */ CScript COINBASE_FLAGS; @@ -100,40 +235,12 @@ const std::string strMessageMagic = "Bitcoin Signed Message:\n"; // Internal stuff namespace { + CBlockIndex *&pindexBestInvalid = g_chainstate.pindexBestInvalid; - struct CBlockIndexWorkComparator - { - bool operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { - // First sort by most total work, ... - if (pa->nChainWork > pb->nChainWork) return false; - if (pa->nChainWork < pb->nChainWork) return true; - - // ... then by earliest time received, ... - if (pa->nSequenceId < pb->nSequenceId) return false; - if (pa->nSequenceId > pb->nSequenceId) return true; - - // Use pointer address as tie breaker (should only happen with blocks - // loaded from disk, as those all have id 0). - if (pa < pb) return false; - if (pa > pb) return true; - - // Identical blocks. - return false; - } - }; - - CBlockIndex *pindexBestInvalid; - - /** - * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and - * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be - * missing the data for the block. - */ - std::set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; /** All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions. * Pruned nodes may have entries where B is missing data. */ - std::multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; + std::multimap<CBlockIndex*, CBlockIndex*>& mapBlocksUnlinked = g_chainstate.mapBlocksUnlinked; CCriticalSection cs_LastBlockFile; std::vector<CBlockFileInfo> vinfoBlockFile; @@ -144,18 +251,6 @@ namespace { */ bool fCheckForPruning = false; - /** - * Every received block is assigned a unique and increasing identifier, so we - * know which one to give priority in case of a fork. - */ - CCriticalSection cs_nBlockSequenceId; - /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ - int32_t nBlockSequenceId = 1; - /** Decreasing counter (used by subsequent preciousblock calls). */ - int32_t nBlockReverseSequenceId = -1; - /** chainwork for the last block that preciousblock has been applied to. */ - arith_uint256 nLastPreciousChainwork = 0; - /** Dirty block index entries. */ std::set<CBlockIndex*> setDirtyBlockIndex; @@ -181,9 +276,9 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc return chain.Genesis(); } -CCoinsViewDB *pcoinsdbview = nullptr; -CCoinsViewCache *pcoinsTip = nullptr; -CBlockTreeDB *pblocktree = nullptr; +std::unique_ptr<CCoinsViewDB> pcoinsdbview; +std::unique_ptr<CCoinsViewCache> pcoinsTip; +std::unique_ptr<CBlockTreeDB> pblocktree; enum FlushStateMode { FLUSH_STATE_NONE, @@ -275,7 +370,7 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool } else { // pcoinsTip contains the UTXO set for chainActive.Tip() - CCoinsViewMemPool viewMemPool(pcoinsTip, mempool); + CCoinsViewMemPool viewMemPool(pcoinsTip.get(), mempool); std::vector<int> prevheights; prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { @@ -404,7 +499,7 @@ void UpdateMempoolForReorg(DisconnectedBlockTransactions &disconnectpool, bool f mempool.UpdateTransactionsFromBlock(vHashUpdate); // We also need to remove any now-immature transactions - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + mempool.removeForReorg(pcoinsTip.get(), chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); // Re-limit mempool size, in case we added any transactions LimitMempoolSize(mempool, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); } @@ -534,11 +629,10 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool CCoinsView dummy; CCoinsViewCache view(&dummy); - CAmount nValueIn = 0; LockPoints lp; { LOCK(pool.cs); - CCoinsViewMemPool viewMemPool(pcoinsTip, pool); + CCoinsViewMemPool viewMemPool(pcoinsTip.get(), pool); view.SetBackend(viewMemPool); // do all inputs exist? @@ -565,8 +659,6 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool // Bring the best block into scope view.GetBestBlock(); - nValueIn = view.GetValueIn(tx); - // we have all inputs cached now, so switch back to dummy, so we don't need to keep lock on mempool view.SetBackend(dummy); @@ -577,6 +669,12 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool // CoinsViewCache instead of create its own if (!CheckSequenceLocks(tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) return state.DoS(0, false, REJECT_NONSTANDARD, "non-BIP68-final"); + + } // end LOCK(pool.cs) + + CAmount nFees = 0; + if (!Consensus::CheckTxInputs(tx, state, view, GetSpendHeight(view), nFees)) { + return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHash().ToString(), FormatStateMessage(state)); } // Check for non-standard pay-to-script-hash in inputs @@ -589,8 +687,6 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool int64_t nSigOpsCost = GetTransactionSigOpCost(tx, view, STANDARD_SCRIPT_VERIFY_FLAGS); - CAmount nValueOut = tx.GetValueOut(); - CAmount nFees = nValueIn-nValueOut; // nModifiedFees includes any fee deltas from PrioritiseTransaction CAmount nModifiedFees = nFees; pool.ApplyDelta(hash, nModifiedFees); @@ -905,44 +1001,51 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return AcceptToMemoryPoolWithTime(chainparams, pool, state, tx, pfMissingInputs, GetTime(), plTxnReplaced, bypass_limits, nAbsurdFee); } -/** Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock */ -bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) +/** + * Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock. + * If blockIndex is provided, the transaction is fetched from the corresponding block. + */ +bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus::Params& consensusParams, uint256& hashBlock, bool fAllowSlow, CBlockIndex* blockIndex) { - CBlockIndex *pindexSlow = nullptr; + CBlockIndex* pindexSlow = blockIndex; LOCK(cs_main); - CTransactionRef ptx = mempool.get(hash); - if (ptx) - { - txOut = ptx; - return true; - } + if (!blockIndex) { + CTransactionRef ptx = mempool.get(hash); + if (ptx) { + txOut = ptx; + return true; + } - if (fTxIndex) { - CDiskTxPos postx; - if (pblocktree->ReadTxIndex(hash, postx)) { - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); - if (file.IsNull()) - return error("%s: OpenBlockFile failed", __func__); - CBlockHeader header; - try { - file >> header; - fseek(file.Get(), postx.nTxOffset, SEEK_CUR); - file >> txOut; - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + if (fTxIndex) { + CDiskTxPos postx; + if (pblocktree->ReadTxIndex(hash, postx)) { + CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) + return error("%s: OpenBlockFile failed", __func__); + CBlockHeader header; + try { + file >> header; + fseek(file.Get(), postx.nTxOffset, SEEK_CUR); + file >> txOut; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + hashBlock = header.GetHash(); + if (txOut->GetHash() != hash) + return error("%s: txid mismatch", __func__); + return true; } - hashBlock = header.GetHash(); - if (txOut->GetHash() != hash) - return error("%s: txid mismatch", __func__); - return true; + + // transaction not found in index, nothing more can be done + return false; } - } - if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - const Coin& coin = AccessByTxid(*pcoinsTip, hash); - if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight]; + if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it + const Coin& coin = AccessByTxid(*pcoinsTip, hash); + if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight]; + } } if (pindexSlow) { @@ -1173,9 +1276,10 @@ void static InvalidChainFound(CBlockIndex* pindexNew) CheckForkWarningConditions(); } -void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { +void CChainState::InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) { if (!state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; + g_failed_blocks.insert(pindex); setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); InvalidChainFound(pindex); @@ -1247,9 +1351,6 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi { if (!tx.IsCoinBase()) { - if (!Consensus::CheckTxInputs(tx, state, inputs, GetSpendHeight(inputs))) - return false; - if (pvChecks) pvChecks->reserve(tx.vin.size()); @@ -1358,8 +1459,13 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint return true; } -bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uint256& hashBlock) +static bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex) { + CDiskBlockPos pos = pindex->GetUndoPos(); + if (pos.IsNull()) { + return error("%s: no undo data available", __func__); + } + // Open history file to read CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) @@ -1369,7 +1475,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin uint256 hashChecksum; CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data try { - verifier << hashBlock; + verifier << pindex->pprev->GetBlockHash(); verifier >> blockundo; filein >> hashChecksum; } @@ -1404,13 +1510,6 @@ bool AbortNode(CValidationState& state, const std::string& strMessage, const std } // namespace -enum DisconnectResult -{ - DISCONNECT_OK, // All good. - DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. - DISCONNECT_FAILED // Something else went wrong. -}; - /** * Restore the UTXO in a Coin at a given COutPoint * @param undo The Coin to be restored. @@ -1447,17 +1546,12 @@ int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) /** Undo the effects of this block (with given index) on the UTXO set represented by coins. * When FAILED is returned, view is left in an indeterminate state. */ -static DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) +DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) { bool fClean = true; CBlockUndo blockUndo; - CDiskBlockPos pos = pindex->GetUndoPos(); - if (pos.IsNull()) { - error("DisconnectBlock(): no undo data available"); - return DISCONNECT_FAILED; - } - if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) { + if (!UndoReadFromDisk(blockUndo, pindex)) { error("DisconnectBlock(): failure reading undo data"); return DISCONNECT_FAILED; } @@ -1534,6 +1628,45 @@ void static FlushBlockFile(bool fFinalize = false) static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); +static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +{ + // Write undo information to disk + if (pindex->GetUndoPos().IsNull()) { + CDiskBlockPos _pos; + if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40)) + return error("ConnectBlock(): FindUndoPos failed"); + if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) + return AbortNode(state, "Failed to write undo data"); + + // update nUndoPos in block index + pindex->nUndoPos = _pos.nPos; + pindex->nStatus |= BLOCK_HAVE_UNDO; + setDirtyBlockIndex.insert(pindex); + } + + return true; +} + +static bool WriteTxIndexDataForBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex) +{ + if (!fTxIndex) return true; + + CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); + std::vector<std::pair<uint256, CDiskTxPos> > vPos; + vPos.reserve(block.vtx.size()); + for (const CTransactionRef& tx : block.vtx) + { + vPos.push_back(std::make_pair(tx->GetHash(), pos)); + pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION); + } + + if (!pblocktree->WriteTxIndex(vPos)) { + return AbortNode(state, "Failed to write transaction index"); + } + + return true; +} + static CCheckQueue<CScriptCheck> scriptcheckqueue(128); void ThreadScriptCheck() { @@ -1589,11 +1722,12 @@ static ThresholdConditionCache warningcache[VERSIONBITS_NUM_BITS]; static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& consensusparams) { AssertLockHeld(cs_main); - // BIP16 didn't become active until Apr 1 2012 - int64_t nBIP16SwitchTime = 1333238400; - bool fStrictPayToScriptHash = (pindex->GetBlockTime() >= nBIP16SwitchTime); + unsigned int flags = SCRIPT_VERIFY_NONE; - unsigned int flags = fStrictPayToScriptHash ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE; + // Start enforcing P2SH (BIP16) + if (pindex->nHeight >= consensusparams.BIP16Height) { + flags |= SCRIPT_VERIFY_P2SH; + } // Start enforcing the DERSIG (BIP66) rule if (pindex->nHeight >= consensusparams.BIP66Height) { @@ -1633,8 +1767,8 @@ static int64_t nBlocksTotal = 0; /** Apply the effects of this block (with given index) on the UTXO set represented by coins. * Validity checks that depend on the UTXO set are also done; ConnectBlock() * can fail if those validity checks fail (among other reasons). */ -static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) +bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) { AssertLockHeld(cs_main); assert(pindex); @@ -1644,6 +1778,18 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd int64_t nTimeStart = GetTimeMicros(); // Check it again in case a previous version let a bad block in + // NOTE: We don't currently (re-)invoke ContextualCheckBlock() or + // ContextualCheckBlockHeader() here. This means that if we add a new + // consensus rule that is enforced in one of those two functions, then we + // may have let in a block that violates the rule prior to updating the + // software, and we would NOT be enforcing the rule here. Fully solving + // upgrade from one software version to the next after a consensus rule + // change is potentially tricky and issue-specific (see RewindBlockIndex() + // for one general approach that was used for BIP 141 deployment). + // Also, currently the rule against blocks more than 2 hours in the future + // is enforced in ContextualCheckBlockHeader(); we wouldn't want to + // re-enforce that rule here (at least until we make it impossible for + // GetAdjustedTime() to go backward). if (!CheckBlock(block, state, chainparams.GetConsensus(), !fJustCheck, !fJustCheck)) return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); @@ -1748,9 +1894,6 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd CAmount nFees = 0; int nInputs = 0; int64_t nSigOpsCost = 0; - CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); - std::vector<std::pair<uint256, CDiskTxPos> > vPos; - vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); std::vector<PrecomputedTransactionData> txdata; txdata.reserve(block.vtx.size()); // Required so that pointers to individual PrecomputedTransactionData don't get invalidated @@ -1762,9 +1905,15 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd if (!tx.IsCoinBase()) { - if (!view.HaveInputs(tx)) - return state.DoS(100, error("ConnectBlock(): inputs missing/spent"), - REJECT_INVALID, "bad-txns-inputs-missingorspent"); + CAmount txfee = 0; + if (!Consensus::CheckTxInputs(tx, state, view, pindex->nHeight, txfee)) { + return error("%s: Consensus::CheckTxInputs: %s, %s", __func__, tx.GetHash().ToString(), FormatStateMessage(state)); + } + nFees += txfee; + if (!MoneyRange(nFees)) { + return state.DoS(100, error("%s: accumulated fee in the block out of range.", __func__), + REJECT_INVALID, "bad-txns-accumulated-fee-outofrange"); + } // Check that transaction is BIP68 final // BIP68 lock checks (as opposed to nLockTime checks) must @@ -1792,8 +1941,6 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd txdata.emplace_back(tx); if (!tx.IsCoinBase()) { - nFees += view.GetValueIn(tx)-tx.GetValueOut(); - std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ if (!CheckInputs(tx, state, view, fScriptChecks, flags, fCacheResults, fCacheResults, txdata[i], nScriptCheckThreads ? &vChecks : nullptr)) @@ -1807,9 +1954,6 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd blockundo.vtxundo.push_back(CTxUndo()); } UpdateCoins(tx, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); - - vPos.push_back(std::make_pair(tx.GetHash(), pos)); - pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); } int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal); @@ -1829,28 +1973,16 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd if (fJustCheck) return true; - // Write undo information to disk - if (pindex->GetUndoPos().IsNull() || !pindex->IsValid(BLOCK_VALID_SCRIPTS)) - { - if (pindex->GetUndoPos().IsNull()) { - CDiskBlockPos _pos; - if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40)) - return error("ConnectBlock(): FindUndoPos failed"); - if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) - return AbortNode(state, "Failed to write undo data"); - - // update nUndoPos in block index - pindex->nUndoPos = _pos.nPos; - pindex->nStatus |= BLOCK_HAVE_UNDO; - } + if (!WriteUndoDataForBlock(blockundo, state, pindex, chainparams)) + return false; + if (!pindex->IsValid(BLOCK_VALID_SCRIPTS)) { pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); setDirtyBlockIndex.insert(pindex); } - if (fTxIndex) - if (!pblocktree->WriteTxIndex(vPos)) - return AbortNode(state, "Failed to write transaction index"); + if (!WriteTxIndexDataForBlock(block, state, pindex)) + return false; assert(pindex->phashBlock); // add this block to the view's block chain @@ -2002,10 +2134,8 @@ static void DoWarning(const std::string& strWarning) } } -/** Update chainActive and related internal data structures. */ -void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { - chainActive.SetTip(pindexNew); - +/** Check warning conditions and do some notifications on new chain tip set. */ +void static UpdateTip(const CBlockIndex *pindexNew, const CChainParams& chainParams) { // New best block mempool.AddTransactionsUpdated(1); @@ -2015,7 +2145,7 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { if (!IsInitialBlockDownload()) { int nUpgraded = 0; - const CBlockIndex* pindex = chainActive.Tip(); + const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(bit); ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); @@ -2046,10 +2176,10 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { } } LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, - chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, - log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); + pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, + log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", pindexNew->GetBlockTime()), + GuessVerificationProgress(chainParams.TxData(), pindexNew), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); if (!warningMessages.empty()) LogPrintf(" warning='%s'", boost::algorithm::join(warningMessages, ", ")); LogPrintf("\n"); @@ -2066,7 +2196,7 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { * disconnectpool (note that the caller is responsible for mempool consistency * in any case). */ -bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) +bool CChainState::DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); @@ -2078,7 +2208,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { - CCoinsViewCache view(pcoinsTip); + CCoinsViewCache view(pcoinsTip.get()); assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); @@ -2103,7 +2233,8 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara } } - // Update chainActive and related variables. + chainActive.SetTip(pindexDelete->pprev); + UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: @@ -2188,7 +2319,7 @@ public: * * The block is added to connectTrace if connection succeeds. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) +bool CChainState::ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. @@ -2208,7 +2339,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, int64_t nTime3; LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDisk * MICRO); { - CCoinsViewCache view(pcoinsTip); + CCoinsViewCache view(pcoinsTip.get()); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams); GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { @@ -2232,6 +2363,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); disconnectpool.removeForBlock(blockConnecting.vtx); // Update chainActive & related variables. + chainActive.SetTip(pindexNew); UpdateTip(pindexNew, chainparams); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; @@ -2246,7 +2378,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, * Return the tip of the chain with the most work in it, that isn't * known to be invalid (it's however far from certain to be valid). */ -static CBlockIndex* FindMostWorkChain() { +CBlockIndex* CChainState::FindMostWorkChain() { do { CBlockIndex *pindexNew = nullptr; @@ -2301,7 +2433,7 @@ static CBlockIndex* FindMostWorkChain() { } /** Delete all entries in setBlockIndexCandidates that are worse than the current tip. */ -static void PruneBlockIndexCandidates() { +void CChainState::PruneBlockIndexCandidates() { // Note that we can't delete the current block itself, as we may need to return to it later in case a // reorganization to a better block fails. std::set<CBlockIndex*, CBlockIndexWorkComparator>::iterator it = setBlockIndexCandidates.begin(); @@ -2316,7 +2448,7 @@ static void PruneBlockIndexCandidates() { * Try to make some progress towards making pindexMostWork the active block. * pblock is either nullptr or a pointer to a CBlock corresponding to pindexMostWork. */ -static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) +bool CChainState::ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) { AssertLockHeld(cs_main); const CBlockIndex *pindexOldTip = chainActive.Tip(); @@ -2386,7 +2518,7 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // any disconnected transactions back to the mempool. UpdateMempoolForReorg(disconnectpool, true); } - mempool.check(pcoinsTip); + mempool.check(pcoinsTip.get()); // Callbacks/notifications for a new best chain. if (fInvalidFound) @@ -2423,17 +2555,26 @@ static void NotifyHeaderTip() { * or an activated best chain. pblock is either nullptr or a pointer to a block * that is already loaded (to avoid loading it again from disk). */ -bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { +bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { // Note that while we're often called here from ProcessNewBlock, this is // far from a guarantee. Things in the P2P/RPC will often end up calling // us in the middle of ProcessNewBlock - do not assume pblock is set // sanely for performance or correctness! + AssertLockNotHeld(cs_main); CBlockIndex *pindexMostWork = nullptr; CBlockIndex *pindexNewTip = nullptr; int nStopAtHeight = gArgs.GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); do { boost::this_thread::interruption_point(); + + if (GetMainSignals().CallbacksPending() > 10) { + // Block until the validation queue drains. This should largely + // never happen in normal operation, however may happen during + // reindex, causing memory blowup if we run too far ahead. + SyncWithValidationInterfaceQueue(); + } + if (ShutdownRequested()) break; @@ -2467,7 +2608,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { assert(trace.pblock && trace.pindex); - GetMainSignals().BlockConnected(trace.pblock, trace.pindex, *trace.conflictedTxs); + GetMainSignals().BlockConnected(trace.pblock, trace.pindex, trace.conflictedTxs); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). @@ -2493,9 +2634,11 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, return true; } +bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { + return g_chainstate.ActivateBestChain(state, chainparams, std::move(pblock)); +} - -bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) +bool CChainState::PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) { { LOCK(cs_main); @@ -2521,24 +2664,28 @@ bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIn } } - return ActivateBestChain(state, params); + return ActivateBestChain(state, params, std::shared_ptr<const CBlock>()); +} +bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) { + return g_chainstate.PreciousBlock(state, params, pindex); } -bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) +bool CChainState::InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { AssertLockHeld(cs_main); - // Mark the block itself as invalid. - pindex->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(pindex); - setBlockIndexCandidates.erase(pindex); + // We first disconnect backwards and then mark the blocks as invalid. + // This prevents a case where pruned nodes may fail to invalidateblock + // and be left unable to start as they have no tip candidates (as there + // are no blocks that meet the "have data and are not invalid per + // nStatus" criteria for inclusion in setBlockIndexCandidates). + + bool pindex_was_in_chain = false; + CBlockIndex *invalid_walk_tip = chainActive.Tip(); DisconnectedBlockTransactions disconnectpool; while (chainActive.Contains(pindex)) { - CBlockIndex *pindexWalk = chainActive.Tip(); - pindexWalk->nStatus |= BLOCK_FAILED_CHILD; - setDirtyBlockIndex.insert(pindexWalk); - setBlockIndexCandidates.erase(pindexWalk); + pindex_was_in_chain = true; // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. if (!DisconnectTip(state, chainparams, &disconnectpool)) { @@ -2549,6 +2696,21 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C } } + // Now mark the blocks we just disconnected as descendants invalid + // (note this may not be all descendants). + while (pindex_was_in_chain && invalid_walk_tip != pindex) { + invalid_walk_tip->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk_tip); + setBlockIndexCandidates.erase(invalid_walk_tip); + invalid_walk_tip = invalid_walk_tip->pprev; + } + + // Mark the block itself as invalid. + pindex->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(pindex); + setBlockIndexCandidates.erase(pindex); + g_failed_blocks.insert(pindex); + // DisconnectTip will add transactions to disconnectpool; try to add these // back to the mempool. UpdateMempoolForReorg(disconnectpool, true); @@ -2567,8 +2729,11 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); return true; } +bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { + return g_chainstate.InvalidateBlock(state, chainparams, pindex); +} -bool ResetBlockFailureFlags(CBlockIndex *pindex) { +bool CChainState::ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); int nHeight = pindex->nHeight; @@ -2586,6 +2751,7 @@ bool ResetBlockFailureFlags(CBlockIndex *pindex) { // Reset invalid block marker if it was pointing to one of those. pindexBestInvalid = nullptr; } + g_failed_blocks.erase(it->second); } it++; } @@ -2600,8 +2766,11 @@ bool ResetBlockFailureFlags(CBlockIndex *pindex) { } return true; } +bool ResetBlockFailureFlags(CBlockIndex *pindex) { + return g_chainstate.ResetBlockFailureFlags(pindex); +} -static CBlockIndex* AddToBlockIndex(const CBlockHeader& block) +CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block) { // Check for duplicate uint256 hash = block.GetHash(); @@ -2611,7 +2780,6 @@ static CBlockIndex* AddToBlockIndex(const CBlockHeader& block) // Construct new block index object CBlockIndex* pindexNew = new CBlockIndex(block); - assert(pindexNew); // We assign the sequence id to blocks only when the full data is available, // to avoid miners withholding blocks but broadcasting headers, to get a // competitive advantage. @@ -2637,7 +2805,7 @@ static CBlockIndex* AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -static bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +bool CChainState::ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2685,7 +2853,7 @@ static bool ReceivedBlockTransactions(const CBlock &block, CValidationState& sta return true; } -static bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) +static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { LOCK(cs_LastBlockFile); @@ -2734,7 +2902,7 @@ static bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned i } } else - return state.Error("out of disk space"); + return error("out of disk space"); } } @@ -2909,7 +3077,13 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc /** Context-dependent validity checks. * By "context", we mean only the previous block headers, but not the UTXO - * set; UTXO-related validity checks are done in ConnectBlock(). */ + * set; UTXO-related validity checks are done in ConnectBlock(). + * NOTE: This function is not currently invoked by ConnectBlock(), so we + * should consider upgrade issues if we change which consensus rules are + * enforced in this function (eg by adding a new consensus rule). See comment + * in ConnectBlock(). + * Note that -reindex-chainstate skips the validation that happens here! + */ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) { assert(pindexPrev != nullptr); @@ -2949,6 +3123,12 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationSta return true; } +/** NOTE: This function is not currently invoked by ConnectBlock(), so we + * should consider upgrade issues if we change which consensus rules are + * enforced in this function (eg by adding a new consensus rule). See comment + * in ConnectBlock(). + * Note that -reindex-chainstate skips the validation that happens here! + */ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { const int nHeight = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1; @@ -3030,7 +3210,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c return true; } -static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) +bool CChainState::AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) { AssertLockHeld(cs_main); // Check for duplicate @@ -3062,6 +3242,21 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); + + if (!pindexPrev->IsValid(BLOCK_VALID_SCRIPTS)) { + for (const CBlockIndex* failedit : g_failed_blocks) { + if (pindexPrev->GetAncestor(failedit->nHeight) == failedit) { + assert(failedit->nStatus & BLOCK_FAILED_VALID); + CBlockIndex* invalid_walk = pindexPrev; + while (invalid_walk != failedit) { + invalid_walk->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(invalid_walk); + invalid_walk = invalid_walk->pprev; + } + return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); + } + } + } } if (pindex == nullptr) pindex = AddToBlockIndex(block); @@ -3075,13 +3270,15 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state } // Exposed wrapper for AcceptBlockHeader -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) +bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex, CBlockHeader *first_invalid) { + if (first_invalid != nullptr) first_invalid->SetNull(); { LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast - if (!AcceptBlockHeader(header, state, chainparams, &pindex)) { + if (!g_chainstate.AcceptBlockHeader(header, state, chainparams, &pindex)) { + if (first_invalid) *first_invalid = header; return false; } if (ppindex) { @@ -3094,7 +3291,26 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) +static CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const CDiskBlockPos* dbp) { + unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); + CDiskBlockPos blockPos; + if (dbp != nullptr) + blockPos = *dbp; + if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) { + error("%s: FindBlockPos failed", __func__); + return CDiskBlockPos(); + } + if (dbp == nullptr) { + if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { + AbortNode("Failed to write block"); + return CDiskBlockPos(); + } + } + return blockPos; +} + +/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ +bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; @@ -3111,7 +3327,7 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation // process an unrequested block if it's new and has enough work to // advance our tip, and isn't too many blocks ahead. bool fAlreadyHave = pindex->nStatus & BLOCK_HAVE_DATA; - bool fHasMoreWork = (chainActive.Tip() ? pindex->nChainWork > chainActive.Tip()->nChainWork : true); + bool fHasMoreOrSameWork = (chainActive.Tip() ? pindex->nChainWork >= chainActive.Tip()->nChainWork : true); // Blocks that are too out-of-order needlessly limit the effectiveness of // pruning, because pruning will not delete block files that contain any // blocks which are too close in height to the tip. Apply this test @@ -3128,9 +3344,15 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation // and unrequested blocks. if (fAlreadyHave) return true; if (!fRequested) { // If we didn't ask for it: - if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned - if (!fHasMoreWork) return true; // Don't process less-work chains - if (fTooFarAhead) return true; // Block height is too high + if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned + if (!fHasMoreOrSameWork) return true; // Don't process less-work chains + if (fTooFarAhead) return true; // Block height is too high + + // Protect against DoS attacks from low-work chains. + // If our tip is behind, a peer could try to send us + // low-work blocks on a fake chain that we would never + // request; don't process these. + if (pindex->nChainWork < nMinimumChainWork) return true; } if (fNewBlock) *fNewBlock = true; @@ -3148,19 +3370,13 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation if (!IsInitialBlockDownload() && chainActive.Tip() == pindex->pprev) GetMainSignals().NewPoWValidBlock(pindex, pblock); - int nHeight = pindex->nHeight; - // Write block to history file try { - unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); - CDiskBlockPos blockPos; - if (dbp != nullptr) - blockPos = *dbp; - if (!FindBlockPos(state, blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) - return error("AcceptBlock(): FindBlockPos failed"); - if (dbp == nullptr) - if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) - AbortNode(state, "Failed to write block"); + CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); + if (blockPos.IsNull()) { + state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); + return false; + } if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("AcceptBlock(): ReceivedBlockTransactions failed"); } catch (const std::runtime_error& e) { @@ -3170,11 +3386,15 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation if (fCheckForPruning) FlushStateToDisk(chainparams, state, FLUSH_STATE_NONE); // we just allocated more disk space for block files + CheckBlockIndex(chainparams.GetConsensus()); + return true; } bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool *fNewBlock) { + AssertLockNotHeld(cs_main); + { CBlockIndex *pindex = nullptr; if (fNewBlock) *fNewBlock = false; @@ -3187,9 +3407,8 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons if (ret) { // Store to disk - ret = AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); + ret = g_chainstate.AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); } - CheckBlockIndex(chainparams.GetConsensus()); if (!ret) { GetMainSignals().BlockChecked(*pblock, state); return error("%s: AcceptBlock FAILED (%s)", __func__, state.GetDebugMessage()); @@ -3199,7 +3418,7 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons NotifyHeaderTip(); CValidationState state; // Only used to report errors, not invalidity - ignore it - if (!ActivateBestChain(state, chainparams, pblock)) + if (!g_chainstate.ActivateBestChain(state, chainparams, pblock)) return error("%s: ActivateBestChain failed", __func__); return true; @@ -3209,7 +3428,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, { AssertLockHeld(cs_main); assert(pindexPrev && pindexPrev == chainActive.Tip()); - CCoinsViewCache viewNew(pcoinsTip); + CCoinsViewCache viewNew(pcoinsTip.get()); CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; @@ -3221,7 +3440,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); - if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) + if (!g_chainstate.ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) return false; assert(state.IsValid()); @@ -3249,8 +3468,8 @@ void PruneOneBlockFile(const int fileNumber) { LOCK(cs_LastBlockFile); - for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) { - CBlockIndex* pindex = it->second; + for (const auto& entry : mapBlockIndex) { + CBlockIndex* pindex = entry.second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; pindex->nStatus &= ~BLOCK_HAVE_UNDO; @@ -3398,7 +3617,7 @@ static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fRe return nullptr; fs::path path = GetBlockPosFilename(pos, prefix); fs::create_directories(path.parent_path()); - FILE* file = fsbridge::fopen(path, "rb+"); + FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+"); if (!file && !fReadOnly) file = fsbridge::fopen(path, "wb+"); if (!file) { @@ -3429,7 +3648,7 @@ fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) return GetDataDir() / "blocks" / strprintf("%s%05u.dat", prefix, pos.nFile); } -CBlockIndex * InsertBlockIndex(uint256 hash) +CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash) { if (hash.IsNull()) return nullptr; @@ -3441,17 +3660,15 @@ CBlockIndex * InsertBlockIndex(uint256 hash) // Create new CBlockIndex* pindexNew = new CBlockIndex(); - if (!pindexNew) - throw std::runtime_error(std::string(__func__) + ": new CBlockIndex failed"); mi = mapBlockIndex.insert(std::make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); return pindexNew; } -bool static LoadBlockIndexDB(const CChainParams& chainparams) +bool CChainState::LoadBlockIndex(const Consensus::Params& consensus_params, CBlockTreeDB& blocktree) { - if (!pblocktree->LoadBlockIndexGuts(chainparams.GetConsensus(), InsertBlockIndex)) + if (!blocktree.LoadBlockIndexGuts(consensus_params, [this](const uint256& hash){ return this->InsertBlockIndex(hash); })) return false; boost::this_thread::interruption_point(); @@ -3484,6 +3701,10 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) pindex->nChainTx = pindex->nTx; } } + if (!(pindex->nStatus & BLOCK_FAILED_MASK) && pindex->pprev && (pindex->pprev->nStatus & BLOCK_FAILED_MASK)) { + pindex->nStatus |= BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(pindex); + } if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->nChainTx || pindex->pprev == nullptr)) setBlockIndexCandidates.insert(pindex); if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) @@ -3494,6 +3715,14 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) pindexBestHeader = pindex; } + return true; +} + +bool static LoadBlockIndexDB(const CChainParams& chainparams) +{ + if (!g_chainstate.LoadBlockIndex(chainparams.GetConsensus(), *pblocktree)) + return false; + // Load block file info pblocktree->ReadLastBlockFile(nLastBlockFile); vinfoBlockFile.resize(nLastBlockFile + 1); @@ -3566,7 +3795,7 @@ bool LoadChainTip(const CChainParams& chainparams) return false; chainActive.SetTip(it->second); - PruneBlockIndexCandidates(); + g_chainstate.PruneBlockIndexCandidates(); LogPrintf("Loaded best chain: hashBestChain=%s height=%d date=%s progress=%f\n", chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), @@ -3631,16 +3860,16 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, // check level 2: verify undo validity if (nCheckLevel >= 2 && pindex) { CBlockUndo undo; - CDiskBlockPos pos = pindex->GetUndoPos(); - if (!pos.IsNull()) { - if (!UndoReadFromDisk(undo, pos, pindex->pprev->GetBlockHash())) + if (!pindex->GetUndoPos().IsNull()) { + if (!UndoReadFromDisk(undo, pindex)) { return error("VerifyDB(): *** found bad undo data at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + } } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); - DisconnectResult res = DisconnectBlock(block, pindex, coins); + DisconnectResult res = g_chainstate.DisconnectBlock(block, pindex, coins); if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -3668,7 +3897,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, CBlock block; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!ConnectBlock(block, state, pindex, coins, chainparams)) + if (!g_chainstate.ConnectBlock(block, state, pindex, coins, chainparams)) return error("VerifyDB(): *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } } @@ -3680,7 +3909,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } /** Apply the effects of a block on the utxo cache, ignoring that it may already have been applied. */ -static bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) +bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) { // TODO: merge with ConnectBlock CBlock block; @@ -3700,7 +3929,7 @@ static bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, return true; } -bool ReplayBlocks(const CChainParams& params, CCoinsView* view) +bool CChainState::ReplayBlocks(const CChainParams& params, CCoinsView* view) { LOCK(cs_main); @@ -3765,7 +3994,11 @@ bool ReplayBlocks(const CChainParams& params, CCoinsView* view) return true; } -bool RewindBlockIndex(const CChainParams& params) +bool ReplayBlocks(const CChainParams& params, CCoinsView* view) { + return g_chainstate.ReplayBlocks(params, view); +} + +bool CChainState::RewindBlockIndex(const CChainParams& params) { LOCK(cs_main); @@ -3802,8 +4035,8 @@ bool RewindBlockIndex(const CChainParams& params) // Reduce validity flag and have-data flags. // We do this after actual disconnecting, otherwise we'll end up writing the lack of data // to disk before writing the chainstate, resulting in a failure to continue if interrupted. - for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { - CBlockIndex* pindexIter = it->second; + for (const auto& entry : mapBlockIndex) { + CBlockIndex* pindexIter = entry.second; // Note: If we encounter an insufficiently validated block that // is on chainActive, it must be because we are a pruning node, and @@ -3846,10 +4079,21 @@ bool RewindBlockIndex(const CChainParams& params) PruneBlockIndexCandidates(); CheckBlockIndex(params.GetConsensus()); + } + + return true; +} + +bool RewindBlockIndex(const CChainParams& params) { + if (!g_chainstate.RewindBlockIndex(params)) { + return false; + } + if (chainActive.Tip() != nullptr) { // FlushStateToDisk can possibly read chainActive. Be conservative // and skip it here, we're about to -reindex-chainstate anyway, so // it'll get called a bunch real soon. + CValidationState state; if (!FlushStateToDisk(params, state, FLUSH_STATE_ALWAYS)) { return false; } @@ -3858,13 +4102,18 @@ bool RewindBlockIndex(const CChainParams& params) return true; } +void CChainState::UnloadBlockIndex() { + nBlockSequenceId = 1; + g_failed_blocks.clear(); + setBlockIndexCandidates.clear(); +} + // May NOT be used after any connections are up as much // of the peer-processing logic assumes a consistent // block index state void UnloadBlockIndex() { LOCK(cs_main); - setBlockIndexCandidates.clear(); chainActive.SetTip(nullptr); pindexBestInvalid = nullptr; pindexBestHeader = nullptr; @@ -3872,7 +4121,6 @@ void UnloadBlockIndex() mapBlocksUnlinked.clear(); vinfoBlockFile.clear(); nLastBlockFile = 0; - nBlockSequenceId = 1; setDirtyBlockIndex.clear(); setDirtyFileInfo.clear(); versionbitscache.Clear(); @@ -3885,6 +4133,8 @@ void UnloadBlockIndex() } mapBlockIndex.clear(); fHavePruned = false; + + g_chainstate.UnloadBlockIndex(); } bool LoadBlockIndex(const CChainParams& chainparams) @@ -3912,7 +4162,7 @@ bool LoadBlockIndex(const CChainParams& chainparams) return true; } -bool LoadGenesisBlock(const CChainParams& chainparams) +bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) { LOCK(cs_main); @@ -3925,15 +4175,11 @@ bool LoadGenesisBlock(const CChainParams& chainparams) try { CBlock &block = const_cast<CBlock&>(chainparams.GenesisBlock()); - // Start new block file - unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); - CDiskBlockPos blockPos; - CValidationState state; - if (!FindBlockPos(state, blockPos, nBlockSize+8, 0, block.GetBlockTime())) - return error("%s: FindBlockPos failed", __func__); - if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) + CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); + if (blockPos.IsNull()) return error("%s: writing genesis block to disk failed", __func__); CBlockIndex *pindex = AddToBlockIndex(block); + CValidationState state; if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("%s: genesis block not accepted", __func__); } catch (const std::runtime_error& e) { @@ -3943,6 +4189,11 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return true; } +bool LoadGenesisBlock(const CChainParams& chainparams) +{ + return g_chainstate.LoadGenesisBlock(chainparams); +} + bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) @@ -4003,7 +4254,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) { LOCK(cs_main); CValidationState state; - if (AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) + if (g_chainstate.AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) nLoaded++; if (state.IsError()) break; @@ -4037,7 +4288,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB head.ToString()); LOCK(cs_main); CValidationState dummy; - if (AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) + if (g_chainstate.AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) { nLoaded++; queue.push_back(pblockrecursive->GetHash()); @@ -4060,7 +4311,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB return nLoaded > 0; } -void static CheckBlockIndex(const Consensus::Params& consensusParams) +void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) { if (!fCheckBlockIndex) { return; @@ -4078,8 +4329,8 @@ void static CheckBlockIndex(const Consensus::Params& consensusParams) // Build forward-pointing map of the entire block tree. std::multimap<CBlockIndex*,CBlockIndex*> forward; - for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { - forward.insert(std::make_pair(it->second->pprev, it->second)); + for (auto& entry : mapBlockIndex) { + forward.insert(std::make_pair(entry.second->pprev, entry.second)); } assert(forward.size() == mapBlockIndex.size()); @@ -4288,8 +4539,9 @@ bool LoadMempool(void) } int64_t count = 0; - int64_t skipped = 0; + int64_t expired = 0; int64_t failed = 0; + int64_t already_there = 0; int64_t nNow = GetTime(); try { @@ -4320,10 +4572,18 @@ bool LoadMempool(void) if (state.IsValid()) { ++count; } else { - ++failed; + // mempool may contain the transaction already, e.g. from + // wallet(s) having loaded it while we were processing + // mempool transactions; consider these as valid, instead of + // failed, but mark them as 'already there' + if (mempool.exists(tx->GetHash())) { + ++already_there; + } else { + ++failed; + } } } else { - ++skipped; + ++expired; } if (ShutdownRequested()) return false; @@ -4339,7 +4599,7 @@ bool LoadMempool(void) return false; } - LogPrintf("Imported mempool transactions from disk: %i successes, %i failed, %i expired\n", count, failed, skipped); + LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there\n", count, failed, expired, already_there); return true; } @@ -4393,7 +4653,7 @@ bool DumpMempool(void) } //! Guess how far we are in the verification process at the given block index -double GuessVerificationProgress(const ChainTxData& data, CBlockIndex *pindex) { +double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pindex) { if (pindex == nullptr) return 0.0; diff --git a/src/validation.h b/src/validation.h index 6bc52753c5..99cbfdf1ee 100644 --- a/src/validation.h +++ b/src/validation.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,17 +7,17 @@ #define BITCOIN_VALIDATION_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif -#include "amount.h" -#include "coins.h" -#include "fs.h" -#include "protocol.h" // For CMessageHeader::MessageStartChars -#include "policy/feerate.h" -#include "script/script_error.h" -#include "sync.h" -#include "versionbits.h" +#include <amount.h> +#include <coins.h> +#include <fs.h> +#include <protocol.h> // For CMessageHeader::MessageStartChars +#include <policy/feerate.h> +#include <script/script_error.h> +#include <sync.h> +#include <versionbits.h> #include <algorithm> #include <exception> @@ -159,7 +159,7 @@ extern CCriticalSection cs_main; extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; -extern BlockMap mapBlockIndex; +extern BlockMap& mapBlockIndex; extern uint64_t nLastBlockTx; extern uint64_t nLastBlockWeight; extern const std::string strMessageMagic; @@ -203,6 +203,8 @@ extern bool fPruneMode; extern uint64_t nPruneTarget; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of chainActive.Tip() will not be pruned. */ static const unsigned int MIN_BLOCKS_TO_KEEP = 288; +/** Minimum blocks required to signal NODE_NETWORK_LIMITED */ +static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; static const signed int DEFAULT_CHECKBLOCKS = 6; static const unsigned int DEFAULT_CHECKLEVEL = 3; @@ -247,8 +249,9 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons * @param[out] state This may be set to an Error state if any error occurred processing them * @param[in] chainparams The params for the chain we want to connect to * @param[out] ppindex If set, the pointer will be set to point to the last new block index object for the given headers + * @param[out] first_invalid First header that fails validation, if one exists */ -bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex=nullptr); +bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex=nullptr, CBlockHeader *first_invalid=nullptr); /** Check whether enough disk space is available for an incoming block */ bool CheckDiskSpace(uint64_t nAdditionalBytes = 0); @@ -272,13 +275,13 @@ void ThreadScriptCheck(); /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload(); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ -bool GetTransaction(const uint256 &hash, CTransactionRef &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); +bool GetTransaction(const uint256& hash, CTransactionRef& tx, const Consensus::Params& params, uint256& hashBlock, bool fAllowSlow = false, CBlockIndex* blockIndex = nullptr); /** Find the best known block, and make it the tip of the block chain */ bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ -double GuessVerificationProgress(const ChainTxData& data, CBlockIndex* pindex); +double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pindex); /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); @@ -293,8 +296,6 @@ void PruneOneBlockFile(const int fileNumber); */ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); -/** Create a new block index entry for a given block hash */ -CBlockIndex * InsertBlockIndex(uint256 hash); /** Flush all state, indexes and buffers to disk. */ void FlushStateToDisk(); /** Prune block files and flush state to disk. */ @@ -440,16 +441,16 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C bool ResetBlockFailureFlags(CBlockIndex *pindex); /** The currently-connected chain of blocks (protected by cs_main). */ -extern CChain chainActive; +extern CChain& chainActive; /** Global variable that points to the coins database (protected by cs_main) */ -extern CCoinsViewDB *pcoinsdbview; +extern std::unique_ptr<CCoinsViewDB> pcoinsdbview; /** Global variable that points to the active CCoinsView (protected by cs_main) */ -extern CCoinsViewCache *pcoinsTip; +extern std::unique_ptr<CCoinsViewCache> pcoinsTip; /** Global variable that points to the active block tree (protected by cs_main) */ -extern CBlockTreeDB *pblocktree; +extern std::unique_ptr<CBlockTreeDB> pblocktree; /** * Return the spend height, which is one more than the inputs.GetBestBlock(). diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 281bc04b0a..928df4fa65 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -1,18 +1,21 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "validationinterface.h" +#include <validationinterface.h> -#include "init.h" -#include "primitives/block.h" -#include "scheduler.h" -#include "sync.h" -#include "util.h" +#include <init.h> +#include <primitives/block.h> +#include <scheduler.h> +#include <sync.h> +#include <txmempool.h> +#include <util.h> +#include <validation.h> #include <list> #include <atomic> +#include <future> #include <boost/signals2/signal.hpp> @@ -21,6 +24,7 @@ struct MainSignalsInstance { boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef>&)> BlockConnected; boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; + boost::signals2::signal<void (const CTransactionRef &)> TransactionRemovedFromMempool; boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; boost::signals2::signal<void (const uint256 &)> Inventory; boost::signals2::signal<void (int64_t nBestBlockTime, CConnman* connman)> Broadcast; @@ -47,7 +51,22 @@ void CMainSignals::UnregisterBackgroundSignalScheduler() { } void CMainSignals::FlushBackgroundCallbacks() { - m_internals->m_schedulerClient.EmptyQueue(); + if (m_internals) { + m_internals->m_schedulerClient.EmptyQueue(); + } +} + +size_t CMainSignals::CallbacksPending() { + if (!m_internals) return 0; + return m_internals->m_schedulerClient.CallbacksPending(); +} + +void CMainSignals::RegisterWithMempoolSignals(CTxMemPool& pool) { + pool.NotifyEntryRemoved.connect(boost::bind(&CMainSignals::MempoolEntryRemoved, this, _1, _2)); +} + +void CMainSignals::UnregisterWithMempoolSignals(CTxMemPool& pool) { + pool.NotifyEntryRemoved.disconnect(boost::bind(&CMainSignals::MempoolEntryRemoved, this, _1, _2)); } CMainSignals& GetMainSignals() @@ -60,6 +79,7 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.m_internals->TransactionAddedToMempool.connect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); g_signals.m_internals->BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); g_signals.m_internals->BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); + g_signals.m_internals->TransactionRemovedFromMempool.connect(boost::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, _1)); g_signals.m_internals->SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.m_internals->Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.m_internals->Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); @@ -75,11 +95,15 @@ void UnregisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.m_internals->TransactionAddedToMempool.disconnect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); g_signals.m_internals->BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); g_signals.m_internals->BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); + g_signals.m_internals->TransactionRemovedFromMempool.disconnect(boost::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, _1)); g_signals.m_internals->UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); g_signals.m_internals->NewPoWValidBlock.disconnect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } void UnregisterAllValidationInterfaces() { + if (!g_signals.m_internals) { + return; + } g_signals.m_internals->BlockChecked.disconnect_all_slots(); g_signals.m_internals->Broadcast.disconnect_all_slots(); g_signals.m_internals->Inventory.disconnect_all_slots(); @@ -87,32 +111,67 @@ void UnregisterAllValidationInterfaces() { g_signals.m_internals->TransactionAddedToMempool.disconnect_all_slots(); g_signals.m_internals->BlockConnected.disconnect_all_slots(); g_signals.m_internals->BlockDisconnected.disconnect_all_slots(); + g_signals.m_internals->TransactionRemovedFromMempool.disconnect_all_slots(); g_signals.m_internals->UpdatedBlockTip.disconnect_all_slots(); g_signals.m_internals->NewPoWValidBlock.disconnect_all_slots(); } +void CallFunctionInValidationInterfaceQueue(std::function<void ()> func) { + g_signals.m_internals->m_schedulerClient.AddToProcessQueue(std::move(func)); +} + +void SyncWithValidationInterfaceQueue() { + AssertLockNotHeld(cs_main); + // Block until the validation queue drains + std::promise<void> promise; + CallFunctionInValidationInterfaceQueue([&promise] { + promise.set_value(); + }); + promise.get_future().wait(); +} + +void CMainSignals::MempoolEntryRemoved(CTransactionRef ptx, MemPoolRemovalReason reason) { + if (reason != MemPoolRemovalReason::BLOCK && reason != MemPoolRemovalReason::CONFLICT) { + m_internals->m_schedulerClient.AddToProcessQueue([ptx, this] { + m_internals->TransactionRemovedFromMempool(ptx); + }); + } +} + void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { - m_internals->UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); + m_internals->m_schedulerClient.AddToProcessQueue([pindexNew, pindexFork, fInitialDownload, this] { + m_internals->UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); + }); } void CMainSignals::TransactionAddedToMempool(const CTransactionRef &ptx) { - m_internals->TransactionAddedToMempool(ptx); + m_internals->m_schedulerClient.AddToProcessQueue([ptx, this] { + m_internals->TransactionAddedToMempool(ptx); + }); } -void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { - m_internals->BlockConnected(pblock, pindex, vtxConflicted); +void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>>& pvtxConflicted) { + m_internals->m_schedulerClient.AddToProcessQueue([pblock, pindex, pvtxConflicted, this] { + m_internals->BlockConnected(pblock, pindex, *pvtxConflicted); + }); } void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock> &pblock) { - m_internals->BlockDisconnected(pblock); + m_internals->m_schedulerClient.AddToProcessQueue([pblock, this] { + m_internals->BlockDisconnected(pblock); + }); } void CMainSignals::SetBestChain(const CBlockLocator &locator) { - m_internals->SetBestChain(locator); + m_internals->m_schedulerClient.AddToProcessQueue([locator, this] { + m_internals->SetBestChain(locator); + }); } void CMainSignals::Inventory(const uint256 &hash) { - m_internals->Inventory(hash); + m_internals->m_schedulerClient.AddToProcessQueue([hash, this] { + m_internals->Inventory(hash); + }); } void CMainSignals::Broadcast(int64_t nBestBlockTime, CConnman* connman) { diff --git a/src/validationinterface.h b/src/validationinterface.h index d6da2bc1fd..56ea698a2e 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -1,14 +1,15 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_VALIDATIONINTERFACE_H #define BITCOIN_VALIDATIONINTERFACE_H -#include <memory> +#include <primitives/transaction.h> // CTransaction(Ref) -#include "primitives/transaction.h" // CTransaction(Ref) +#include <functional> +#include <memory> class CBlock; class CBlockIndex; @@ -20,6 +21,8 @@ class CValidationInterface; class CValidationState; class uint256; class CScheduler; +class CTxMemPool; +enum class MemPoolRemovalReason; // These functions dispatch to one or all registered wallets @@ -29,23 +32,76 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn); void UnregisterValidationInterface(CValidationInterface* pwalletIn); /** Unregister all wallets from core */ void UnregisterAllValidationInterfaces(); +/** + * Pushes a function to callback onto the notification queue, guaranteeing any + * callbacks generated prior to now are finished when the function is called. + * + * Be very careful blocking on func to be called if any locks are held - + * validation interface clients may not be able to make progress as they often + * wait for things like cs_main, so blocking until func is called with cs_main + * will result in a deadlock (that DEBUG_LOCKORDER will miss). + */ +void CallFunctionInValidationInterfaceQueue(std::function<void ()> func); +/** + * This is a synonym for the following, which asserts certain locks are not + * held: + * std::promise<void> promise; + * CallFunctionInValidationInterfaceQueue([&promise] { + * promise.set_value(); + * }); + * promise.get_future().wait(); + */ +void SyncWithValidationInterfaceQueue(); class CValidationInterface { protected: - /** Notifies listeners of updated block chain tip */ + /** + * Notifies listeners of updated block chain tip + * + * Called on a background thread. + */ virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} - /** Notifies listeners of a transaction having been added to mempool. */ + /** + * Notifies listeners of a transaction having been added to mempool. + * + * Called on a background thread. + */ virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} /** + * Notifies listeners of a transaction leaving mempool. + * + * This only fires for transactions which leave mempool because of expiry, + * size limiting, reorg (changes in lock times/coinbase maturity), or + * replacement. This does not include any transactions which are included + * in BlockConnectedDisconnected either in block->vtx or in txnConflicted. + * + * Called on a background thread. + */ + virtual void TransactionRemovedFromMempool(const CTransactionRef &ptx) {} + /** * Notifies listeners of a block being connected. * Provides a vector of transactions evicted from the mempool as a result. + * + * Called on a background thread. */ virtual void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex, const std::vector<CTransactionRef> &txnConflicted) {} - /** Notifies listeners of a block being disconnected */ + /** + * Notifies listeners of a block being disconnected + * + * Called on a background thread. + */ virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} - /** Notifies listeners of the new active block chain on-disk. */ + /** + * Notifies listeners of the new active block chain on-disk. + * + * Called on a background thread. + */ virtual void SetBestChain(const CBlockLocator &locator) {} - /** Notifies listeners about an inventory item being seen on the network. */ + /** + * Notifies listeners about an inventory item being seen on the network. + * + * Called on a background thread. + */ virtual void Inventory(const uint256 &hash) {} /** Tells listeners to broadcast their data. */ virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} @@ -73,6 +129,9 @@ private: friend void ::RegisterValidationInterface(CValidationInterface*); friend void ::UnregisterValidationInterface(CValidationInterface*); friend void ::UnregisterAllValidationInterfaces(); + friend void ::CallFunctionInValidationInterfaceQueue(std::function<void ()> func); + + void MempoolEntryRemoved(CTransactionRef tx, MemPoolRemovalReason reason); public: /** Register a CScheduler to give callbacks which should run in the background (may only be called once) */ @@ -82,9 +141,16 @@ public: /** Call any remaining callbacks on the calling thread */ void FlushBackgroundCallbacks(); + size_t CallbacksPending(); + + /** Register with mempool to call TransactionRemovedFromMempool callbacks */ + void RegisterWithMempoolSignals(CTxMemPool& pool); + /** Unregister with mempool */ + void UnregisterWithMempoolSignals(CTxMemPool& pool); + void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); void TransactionAddedToMempool(const CTransactionRef &); - void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef> &); + void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>> &); void BlockDisconnected(const std::shared_ptr<const CBlock> &); void SetBestChain(const CBlockLocator &); void Inventory(const uint256 &); diff --git a/src/version.h b/src/version.h index d528212490..4b38547f35 100644 --- a/src/version.h +++ b/src/version.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/versionbits.cpp b/src/versionbits.cpp index 64ae939672..d2ee49db20 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "versionbits.h" -#include "consensus/params.h" +#include <versionbits.h> +#include <consensus/params.h> const struct VBDeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { { @@ -27,6 +27,11 @@ ThresholdState AbstractThresholdConditionChecker::GetStateFor(const CBlockIndex* int64_t nTimeStart = BeginTime(params); int64_t nTimeTimeout = EndTime(params); + // Check if this deployment is always active. + if (nTimeStart == Consensus::BIP9Deployment::ALWAYS_ACTIVE) { + return THRESHOLD_ACTIVE; + } + // A block's state is always the same as that of the first of its period, so it is computed based on a pindexPrev whose height equals a multiple of nPeriod - 1. if (pindexPrev != nullptr) { pindexPrev = pindexPrev->GetAncestor(pindexPrev->nHeight - ((pindexPrev->nHeight + 1) % nPeriod)); @@ -136,6 +141,11 @@ BIP9Stats AbstractThresholdConditionChecker::GetStateStatisticsFor(const CBlockI int AbstractThresholdConditionChecker::GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const { + int64_t start_time = BeginTime(params); + if (start_time == Consensus::BIP9Deployment::ALWAYS_ACTIVE) { + return 0; + } + const ThresholdState initialState = GetStateFor(pindexPrev, params, cache); // BIP 9 about state DEFINED: "The genesis block is by definition in this state for each deployment." diff --git a/src/versionbits.h b/src/versionbits.h index 71dc0c8500..1600dc8c93 100644 --- a/src/versionbits.h +++ b/src/versionbits.h @@ -1,11 +1,11 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CONSENSUS_VERSIONBITS #define BITCOIN_CONSENSUS_VERSIONBITS -#include "chain.h" +#include <chain.h> #include <map> /** What block version to use for new blocks (pre versionbits) */ diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index fc0e7c519e..e1afa2de03 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -1,14 +1,14 @@ -// Copyright (c) 2011-2016 The Bitcoin Core developers +// Copyright (c) 2011-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_WALLET_COINCONTROL_H #define BITCOIN_WALLET_COINCONTROL_H -#include "policy/feerate.h" -#include "policy/fees.h" -#include "primitives/transaction.h" -#include "wallet/wallet.h" +#include <policy/feerate.h> +#include <policy/fees.h> +#include <primitives/transaction.h> +#include <wallet/wallet.h> #include <boost/optional.hpp> diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 8db3bfd69c..6ad18721fd 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "crypter.h" +#include <wallet/crypter.h> -#include "crypto/aes.h" -#include "crypto/sha512.h" -#include "script/script.h" -#include "script/standard.h" -#include "util.h" +#include <crypto/aes.h> +#include <crypto/sha512.h> +#include <script/script.h> +#include <script/standard.h> +#include <util.h> #include <string> #include <vector> @@ -152,6 +152,15 @@ bool CCryptoKeyStore::SetCrypted() return true; } +bool CCryptoKeyStore::IsLocked() const +{ + if (!IsCrypted()) { + return false; + } + LOCK(cs_KeyStore); + return vMasterKey.empty(); +} + bool CCryptoKeyStore::Lock() { if (!SetCrypted()) @@ -206,21 +215,23 @@ bool CCryptoKeyStore::Unlock(const CKeyingMaterial& vMasterKeyIn) bool CCryptoKeyStore::AddKeyPubKey(const CKey& key, const CPubKey &pubkey) { - { - LOCK(cs_KeyStore); - if (!IsCrypted()) - return CBasicKeyStore::AddKeyPubKey(key, pubkey); + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return CBasicKeyStore::AddKeyPubKey(key, pubkey); + } - if (IsLocked()) - return false; + if (IsLocked()) { + return false; + } - std::vector<unsigned char> vchCryptedSecret; - CKeyingMaterial vchSecret(key.begin(), key.end()); - if (!EncryptSecret(vMasterKey, vchSecret, pubkey.GetHash(), vchCryptedSecret)) - return false; + std::vector<unsigned char> vchCryptedSecret; + CKeyingMaterial vchSecret(key.begin(), key.end()); + if (!EncryptSecret(vMasterKey, vchSecret, pubkey.GetHash(), vchCryptedSecret)) { + return false; + } - if (!AddCryptedKey(pubkey, vchCryptedSecret)) - return false; + if (!AddCryptedKey(pubkey, vchCryptedSecret)) { + return false; } return true; } @@ -228,72 +239,89 @@ bool CCryptoKeyStore::AddKeyPubKey(const CKey& key, const CPubKey &pubkey) bool CCryptoKeyStore::AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) { - { - LOCK(cs_KeyStore); - if (!SetCrypted()) - return false; - - mapCryptedKeys[vchPubKey.GetID()] = make_pair(vchPubKey, vchCryptedSecret); + LOCK(cs_KeyStore); + if (!SetCrypted()) { + return false; } + + mapCryptedKeys[vchPubKey.GetID()] = make_pair(vchPubKey, vchCryptedSecret); + ImplicitlyLearnRelatedKeyScripts(vchPubKey); return true; } +bool CCryptoKeyStore::HaveKey(const CKeyID &address) const +{ + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return CBasicKeyStore::HaveKey(address); + } + return mapCryptedKeys.count(address) > 0; +} + bool CCryptoKeyStore::GetKey(const CKeyID &address, CKey& keyOut) const { - { - LOCK(cs_KeyStore); - if (!IsCrypted()) - return CBasicKeyStore::GetKey(address, keyOut); + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return CBasicKeyStore::GetKey(address, keyOut); + } - CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); - if (mi != mapCryptedKeys.end()) - { - const CPubKey &vchPubKey = (*mi).second.first; - const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; - return DecryptKey(vMasterKey, vchCryptedSecret, vchPubKey, keyOut); - } + CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); + if (mi != mapCryptedKeys.end()) + { + const CPubKey &vchPubKey = (*mi).second.first; + const std::vector<unsigned char> &vchCryptedSecret = (*mi).second.second; + return DecryptKey(vMasterKey, vchCryptedSecret, vchPubKey, keyOut); } return false; } bool CCryptoKeyStore::GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const { + LOCK(cs_KeyStore); + if (!IsCrypted()) + return CBasicKeyStore::GetPubKey(address, vchPubKeyOut); + + CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); + if (mi != mapCryptedKeys.end()) { - LOCK(cs_KeyStore); - if (!IsCrypted()) - return CBasicKeyStore::GetPubKey(address, vchPubKeyOut); + vchPubKeyOut = (*mi).second.first; + return true; + } + // Check for watch-only pubkeys + return CBasicKeyStore::GetPubKey(address, vchPubKeyOut); +} - CryptedKeyMap::const_iterator mi = mapCryptedKeys.find(address); - if (mi != mapCryptedKeys.end()) - { - vchPubKeyOut = (*mi).second.first; - return true; - } - // Check for watch-only pubkeys - return CBasicKeyStore::GetPubKey(address, vchPubKeyOut); +std::set<CKeyID> CCryptoKeyStore::GetKeys() const +{ + LOCK(cs_KeyStore); + if (!IsCrypted()) { + return CBasicKeyStore::GetKeys(); } + std::set<CKeyID> set_address; + for (const auto& mi : mapCryptedKeys) { + set_address.insert(mi.first); + } + return set_address; } bool CCryptoKeyStore::EncryptKeys(CKeyingMaterial& vMasterKeyIn) { + LOCK(cs_KeyStore); + if (!mapCryptedKeys.empty() || IsCrypted()) + return false; + + fUseCrypto = true; + for (KeyMap::value_type& mKey : mapKeys) { - LOCK(cs_KeyStore); - if (!mapCryptedKeys.empty() || IsCrypted()) + const CKey &key = mKey.second; + CPubKey vchPubKey = key.GetPubKey(); + CKeyingMaterial vchSecret(key.begin(), key.end()); + std::vector<unsigned char> vchCryptedSecret; + if (!EncryptSecret(vMasterKeyIn, vchSecret, vchPubKey.GetHash(), vchCryptedSecret)) + return false; + if (!AddCryptedKey(vchPubKey, vchCryptedSecret)) return false; - - fUseCrypto = true; - for (KeyMap::value_type& mKey : mapKeys) - { - const CKey &key = mKey.second; - CPubKey vchPubKey = key.GetPubKey(); - CKeyingMaterial vchSecret(key.begin(), key.end()); - std::vector<unsigned char> vchCryptedSecret; - if (!EncryptSecret(vMasterKeyIn, vchSecret, vchPubKey.GetHash(), vchCryptedSecret)) - return false; - if (!AddCryptedKey(vchPubKey, vchCryptedSecret)) - return false; - } - mapKeys.clear(); } + mapKeys.clear(); return true; } diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index 1416ae7d02..f3ae7144b4 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -1,13 +1,13 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_WALLET_CRYPTER_H #define BITCOIN_WALLET_CRYPTER_H -#include "keystore.h" -#include "serialize.h" -#include "support/allocators/secure.h" +#include <keystore.h> +#include <serialize.h> +#include <support/allocators/secure.h> #include <atomic> @@ -139,52 +139,16 @@ public: { } - bool IsCrypted() const - { - return fUseCrypto; - } - - bool IsLocked() const - { - if (!IsCrypted()) - return false; - bool result; - { - LOCK(cs_KeyStore); - result = vMasterKey.empty(); - } - return result; - } - + bool IsCrypted() const { return fUseCrypto; } + bool IsLocked() const; bool Lock(); virtual bool AddCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret); bool AddKeyPubKey(const CKey& key, const CPubKey &pubkey) override; - bool HaveKey(const CKeyID &address) const override - { - { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return CBasicKeyStore::HaveKey(address); - } - return mapCryptedKeys.count(address) > 0; - } - return false; - } + bool HaveKey(const CKeyID &address) const override; bool GetKey(const CKeyID &address, CKey& keyOut) const override; bool GetPubKey(const CKeyID &address, CPubKey& vchPubKeyOut) const override; - std::set<CKeyID> GetKeys() const override - { - LOCK(cs_KeyStore); - if (!IsCrypted()) { - return CBasicKeyStore::GetKeys(); - } - std::set<CKeyID> set_address; - for (const auto& mi : mapCryptedKeys) { - set_address.insert(mi.first); - } - return set_address; - } + std::set<CKeyID> GetKeys() const override; /** * Wallet status (encrypted, locked) changed. diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index d66ba48421..35ff0e1eec 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -1,16 +1,16 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "db.h" +#include <wallet/db.h> -#include "addrman.h" -#include "fs.h" -#include "hash.h" -#include "protocol.h" -#include "util.h" -#include "utilstrencodings.h" +#include <addrman.h> +#include <hash.h> +#include <protocol.h> +#include <util.h> +#include <utilstrencodings.h> +#include <wallet/walletutil.h> #include <stdint.h> @@ -20,6 +20,40 @@ #include <boost/thread.hpp> +namespace { +//! Make sure database has a unique fileid within the environment. If it +//! doesn't, throw an error. BDB caches do not work properly when more than one +//! open database has the same fileid (values written to one database may show +//! up in reads to other databases). +//! +//! BerkeleyDB generates unique fileids by default +//! (https://docs.oracle.com/cd/E17275_01/html/programmer_reference/program_copy.html), +//! so bitcoin should never create different databases with the same fileid, but +//! this error can be triggered if users manually copy database files. +void CheckUniqueFileid(const CDBEnv& env, const std::string& filename, Db& db) +{ + if (env.IsMock()) return; + + u_int8_t fileid[DB_FILE_ID_LEN]; + int ret = db.get_mpf()->get_fileid(fileid); + if (ret != 0) { + throw std::runtime_error(strprintf("CDB: Can't open database %s (get_fileid failed with %d)", filename, ret)); + } + + for (const auto& item : env.mapDb) { + u_int8_t item_fileid[DB_FILE_ID_LEN]; + if (item.second && item.second->get_mpf()->get_fileid(item_fileid) == 0 && + memcmp(fileid, item_fileid, sizeof(fileid)) == 0) { + const char* item_filename = nullptr; + item.second->get_dbname(&item_filename, nullptr); + throw std::runtime_error(strprintf("CDB: Can't open database %s (duplicates fileid %s from %s)", filename, + HexStr(std::begin(item_fileid), std::end(item_fileid)), + item_filename ? item_filename : "(unknown database)")); + } + } +} +} // namespace + // // CDB // @@ -41,13 +75,12 @@ void CDBEnv::EnvShutdown() void CDBEnv::Reset() { - delete dbenv; - dbenv = new DbEnv(DB_CXX_NO_EXCEPTIONS); + dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS)); fDbEnvInit = false; fMockDb = false; } -CDBEnv::CDBEnv() : dbenv(nullptr) +CDBEnv::CDBEnv() { Reset(); } @@ -55,8 +88,6 @@ CDBEnv::CDBEnv() : dbenv(nullptr) CDBEnv::~CDBEnv() { EnvShutdown(); - delete dbenv; - dbenv = nullptr; } void CDBEnv::Close() @@ -148,7 +179,7 @@ CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, recoverFunc_type LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); - Db db(dbenv, 0); + Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); if (result == 0) return VERIFY_OK; @@ -191,7 +222,7 @@ bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*reco } LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); - std::unique_ptr<Db> pdbCopy(new Db(bitdb.dbenv, 0)); + std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(bitdb.dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer filename.c_str(), // Filename "main", // Logical db name @@ -226,7 +257,7 @@ bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*reco return fSuccess; } -bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) +bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr) { LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); LogPrintf("Using wallet %s\n", walletFile); @@ -234,15 +265,15 @@ bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataD // Wallet file must be a plain filename without a directory if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) { - errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string()); + errorStr = strprintf(_("Wallet %s resides outside wallet directory %s"), walletFile, walletDir.string()); return false; } - if (!bitdb.Open(dataDir)) + if (!bitdb.Open(walletDir)) { // try moving the database env out of the way - fs::path pathDatabase = dataDir / "database"; - fs::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime()); + fs::path pathDatabase = walletDir / "database"; + fs::path pathDatabaseBak = walletDir / strprintf("database.%d.bak", GetTime()); try { fs::rename(pathDatabase, pathDatabaseBak); LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); @@ -251,18 +282,18 @@ bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataD } // try again - if (!bitdb.Open(dataDir)) { + if (!bitdb.Open(walletDir)) { // if it still fails, it probably means we can't even create the database env - errorStr = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir()); + errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir); return false; } } return true; } -bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc) +bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc) { - if (fs::exists(dataDir / walletFile)) + if (fs::exists(walletDir / walletFile)) { std::string backup_filename; CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc, backup_filename); @@ -272,7 +303,7 @@ bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& data " Original %s saved as %s in %s; if" " your balance or transactions are incorrect you should" " restore from a backup."), - walletFile, backup_filename, dataDir); + walletFile, backup_filename, walletDir); } if (r == CDBEnv::RECOVER_FAIL) { @@ -300,7 +331,7 @@ bool CDBEnv::Salvage(const std::string& strFile, bool fAggressive, std::vector<C std::stringstream strDump; - Db db(dbenv, 0); + Db db(dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, &strDump, flags); if (result == DB_VERIFY_BAD) { LogPrintf("CDBEnv::Salvage: Database salvage found errors, all data may not be recoverable.\n"); @@ -376,38 +407,37 @@ CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb { LOCK(env->cs_db); - if (!env->Open(GetDataDir())) + if (!env->Open(GetWalletDir())) throw std::runtime_error("CDB: Failed to open database environment."); - strFile = strFilename; - ++env->mapFileUseCount[strFile]; - pdb = env->mapDb[strFile]; + pdb = env->mapDb[strFilename]; if (pdb == nullptr) { int ret; - pdb = new Db(env->dbenv, 0); + std::unique_ptr<Db> pdb_temp = MakeUnique<Db>(env->dbenv.get(), 0); bool fMockDb = env->IsMock(); if (fMockDb) { - DbMpoolFile* mpf = pdb->get_mpf(); + DbMpoolFile* mpf = pdb_temp->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); - if (ret != 0) - throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFile)); + if (ret != 0) { + throw std::runtime_error(strprintf("CDB: Failed to configure for no temp file backing for database %s", strFilename)); + } } - ret = pdb->open(nullptr, // Txn pointer - fMockDb ? nullptr : strFile.c_str(), // Filename - fMockDb ? strFile.c_str() : "main", // Logical db name - DB_BTREE, // Database type - nFlags, // Flags + ret = pdb_temp->open(nullptr, // Txn pointer + fMockDb ? nullptr : strFilename.c_str(), // Filename + fMockDb ? strFilename.c_str() : "main", // Logical db name + DB_BTREE, // Database type + nFlags, // Flags 0); if (ret != 0) { - delete pdb; - pdb = nullptr; - --env->mapFileUseCount[strFile]; - strFile = ""; throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename)); } + CheckUniqueFileid(*env, strFilename, *pdb_temp); + + pdb = pdb_temp.release(); + env->mapDb[strFilename] = pdb; if (fCreate && !Exists(std::string("version"))) { bool fTmp = fReadOnly; @@ -415,9 +445,9 @@ CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb WriteVersion(CLIENT_VERSION); fReadOnly = fTmp; } - - env->mapDb[strFile] = pdb; } + ++env->mapFileUseCount[strFilename]; + strFile = strFilename; } } @@ -492,7 +522,7 @@ bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip) std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} CDB db(dbw, "r"); - Db* pdbCopy = new Db(env->dbenv, 0); + std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0); int ret = pdbCopy->open(nullptr, // Txn pointer strFileRes.c_str(), // Filename @@ -541,13 +571,12 @@ bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip) } else { pdbCopy->close(0); } - delete pdbCopy; } if (fSuccess) { - Db dbA(env->dbenv, 0); + Db dbA(env->dbenv.get(), 0); if (dbA.remove(strFile.c_str(), nullptr, 0)) fSuccess = false; - Db dbB(env->dbenv, 0); + Db dbB(env->dbenv.get(), 0); if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0)) fSuccess = false; } @@ -666,12 +695,17 @@ bool CWalletDBWrapper::Backup(const std::string& strDest) env->mapFileUseCount.erase(strFile); // Copy wallet file - fs::path pathSrc = GetDataDir() / strFile; + fs::path pathSrc = GetWalletDir() / strFile; fs::path pathDest(strDest); if (fs::is_directory(pathDest)) pathDest /= strFile; try { + if (fs::equivalent(pathSrc, pathDest)) { + LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); + return false; + } + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); LogPrintf("copied %s to %s\n", strFile, pathDest.string()); return true; diff --git a/src/wallet/db.h b/src/wallet/db.h index 14283ac8f8..c6f317927f 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -1,17 +1,17 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_WALLET_DB_H #define BITCOIN_WALLET_DB_H -#include "clientversion.h" -#include "fs.h" -#include "serialize.h" -#include "streams.h" -#include "sync.h" -#include "version.h" +#include <clientversion.h> +#include <fs.h> +#include <serialize.h> +#include <streams.h> +#include <sync.h> +#include <version.h> #include <atomic> #include <map> @@ -36,7 +36,7 @@ private: public: mutable CCriticalSection cs_db; - DbEnv *dbenv; + std::unique_ptr<DbEnv> dbenv; std::map<std::string, int> mapFileUseCount; std::map<std::string, Db*> mapDb; @@ -167,9 +167,9 @@ public: ideal to be called periodically */ static bool PeriodicFlush(CWalletDBWrapper& dbw); /* verifies the database environment */ - static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc); + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc); public: template <typename K, typename T> diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 6abd060714..9bfcab54a5 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -2,19 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "consensus/validation.h" -#include "wallet/coincontrol.h" -#include "wallet/feebumper.h" -#include "wallet/fees.h" -#include "wallet/wallet.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "validation.h" //for mempool access -#include "txmempool.h" -#include "utilmoneystr.h" -#include "util.h" -#include "net.h" +#include <consensus/validation.h> +#include <wallet/coincontrol.h> +#include <wallet/feebumper.h> +#include <wallet/fees.h> +#include <wallet/wallet.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <validation.h> //for mempool access +#include <txmempool.h> +#include <utilmoneystr.h> +#include <util.h> +#include <net.h> // Calculate the size of the transaction assuming all signatures are max size // Use DummySignatureCreator, which inserts 72 byte signatures everywhere. @@ -23,7 +23,7 @@ // calculation, but we should be able to refactor after priority is removed). // NOTE: this requires that all inputs must be in mapWallet (eg the tx should // be IsAllFromMe). -int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWallet) +static int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet) { CMutableTransaction txNew(tx); std::vector<CInputCoin> vCoins; @@ -31,11 +31,11 @@ int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWal // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our // wallet, with a valid index into the vout array. for (auto& input : tx.vin) { - const auto mi = pWallet->mapWallet.find(input.prevout.hash); - assert(mi != pWallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); + const auto mi = wallet->mapWallet.find(input.prevout.hash); + assert(mi != wallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); vCoins.emplace_back(CInputCoin(&(mi->second), input.prevout.n)); } - if (!pWallet->DummySignTx(txNew, vCoins)) { + if (!wallet->DummySignTx(txNew, vCoins)) { // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) // implies that we can sign for every input. return -1; @@ -43,103 +43,102 @@ int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWal return GetVirtualTransactionSize(txNew); } -bool CFeeBumper::preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx) { - if (pWallet->HasWalletSpend(wtx.GetHash())) { - vErrors.push_back("Transaction has descendants in the wallet"); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return false; +//! Check whether transaction has descendant in wallet or mempool, or has been +//! mined, or conflicts with a mined transaction. Return a feebumper::Result. +static feebumper::Result PreconditionChecks(const CWallet* wallet, const CWalletTx& wtx, std::vector<std::string>& errors) +{ + if (wallet->HasWalletSpend(wtx.GetHash())) { + errors.push_back("Transaction has descendants in the wallet"); + return feebumper::Result::INVALID_PARAMETER; } { LOCK(mempool.cs); auto it_mp = mempool.mapTx.find(wtx.GetHash()); if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { - vErrors.push_back("Transaction has descendants in the mempool"); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return false; + errors.push_back("Transaction has descendants in the mempool"); + return feebumper::Result::INVALID_PARAMETER; } } if (wtx.GetDepthInMainChain() != 0) { - vErrors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); - currentResult = BumpFeeResult::WALLET_ERROR; - return false; + errors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); + return feebumper::Result::WALLET_ERROR; } - return true; + return feebumper::Result::OK; } -CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, const CCoinControl& coin_control, CAmount totalFee) - : - txid(std::move(txidIn)), - nOldFee(0), - nNewFee(0) +namespace feebumper { + +bool TransactionCanBeBumped(CWallet* wallet, const uint256& txid) { - vErrors.clear(); - bumpedTxid.SetNull(); - AssertLockHeld(pWallet->cs_wallet); - auto it = pWallet->mapWallet.find(txid); - if (it == pWallet->mapWallet.end()) { - vErrors.push_back("Invalid or non-wallet transaction id"); - currentResult = BumpFeeResult::INVALID_ADDRESS_OR_KEY; - return; + LOCK2(cs_main, wallet->cs_wallet); + const CWalletTx* wtx = wallet->GetWalletTx(txid); + return wtx && SignalsOptInRBF(*wtx->tx) && !wtx->mapValue.count("replaced_by_txid"); +} + +Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoinControl& coin_control, CAmount total_fee, std::vector<std::string>& errors, + CAmount& old_fee, CAmount& new_fee, CMutableTransaction& mtx) +{ + LOCK2(cs_main, wallet->cs_wallet); + errors.clear(); + auto it = wallet->mapWallet.find(txid); + if (it == wallet->mapWallet.end()) { + errors.push_back("Invalid or non-wallet transaction id"); + return Result::INVALID_ADDRESS_OR_KEY; } const CWalletTx& wtx = it->second; - if (!preconditionChecks(pWallet, wtx)) { - return; + Result result = PreconditionChecks(wallet, wtx, errors); + if (result != Result::OK) { + return result; } - if (!SignalsOptInRBF(wtx)) { - vErrors.push_back("Transaction is not BIP 125 replaceable"); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + if (!SignalsOptInRBF(*wtx.tx)) { + errors.push_back("Transaction is not BIP 125 replaceable"); + return Result::WALLET_ERROR; } if (wtx.mapValue.count("replaced_by_txid")) { - vErrors.push_back(strprintf("Cannot bump transaction %s which was already bumped by transaction %s", txid.ToString(), wtx.mapValue.at("replaced_by_txid"))); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + errors.push_back(strprintf("Cannot bump transaction %s which was already bumped by transaction %s", txid.ToString(), wtx.mapValue.at("replaced_by_txid"))); + return Result::WALLET_ERROR; } // check that original tx consists entirely of our inputs // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) - if (!pWallet->IsAllFromMe(wtx, ISMINE_SPENDABLE)) { - vErrors.push_back("Transaction contains inputs that don't belong to this wallet"); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + if (!wallet->IsAllFromMe(*wtx.tx, ISMINE_SPENDABLE)) { + errors.push_back("Transaction contains inputs that don't belong to this wallet"); + return Result::WALLET_ERROR; } // figure out which output was change // if there was no change output or multiple change outputs, fail int nOutput = -1; for (size_t i = 0; i < wtx.tx->vout.size(); ++i) { - if (pWallet->IsChange(wtx.tx->vout[i])) { + if (wallet->IsChange(wtx.tx->vout[i])) { if (nOutput != -1) { - vErrors.push_back("Transaction has multiple change outputs"); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + errors.push_back("Transaction has multiple change outputs"); + return Result::WALLET_ERROR; } nOutput = i; } } if (nOutput == -1) { - vErrors.push_back("Transaction does not have a change output"); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + errors.push_back("Transaction does not have a change output"); + return Result::WALLET_ERROR; } // Calculate the expected size of the new transaction. int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); - const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx, pWallet); + const int64_t maxNewTxSize = CalculateMaximumSignedTxSize(*wtx.tx, wallet); if (maxNewTxSize < 0) { - vErrors.push_back("Transaction contains inputs that cannot be signed"); - currentResult = BumpFeeResult::INVALID_ADDRESS_OR_KEY; - return; + errors.push_back("Transaction contains inputs that cannot be signed"); + return Result::INVALID_ADDRESS_OR_KEY; } // calculate the old fee and fee-rate - nOldFee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); - CFeeRate nOldFeeRate(nOldFee, txSize); + old_fee = wtx.GetDebit(ISMINE_SPENDABLE) - wtx.tx->GetValueOut(); + CFeeRate nOldFeeRate(old_fee, txSize); CFeeRate nNewFeeRate; // The wallet uses a conservative WALLET_INCREMENTAL_RELAY_FEE value to // future proof against changes to network wide policy for incremental relay @@ -149,26 +148,24 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, const CCoin walletIncrementalRelayFee = ::incrementalRelayFee; } - if (totalFee > 0) { + if (total_fee > 0) { CAmount minTotalFee = nOldFeeRate.GetFee(maxNewTxSize) + ::incrementalRelayFee.GetFee(maxNewTxSize); - if (totalFee < minTotalFee) { - vErrors.push_back(strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", + if (total_fee < minTotalFee) { + errors.push_back(strprintf("Insufficient totalFee, must be at least %s (oldFee %s + incrementalFee %s)", FormatMoney(minTotalFee), FormatMoney(nOldFeeRate.GetFee(maxNewTxSize)), FormatMoney(::incrementalRelayFee.GetFee(maxNewTxSize)))); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return; + return Result::INVALID_PARAMETER; } CAmount requiredFee = GetRequiredFee(maxNewTxSize); - if (totalFee < requiredFee) { - vErrors.push_back(strprintf("Insufficient totalFee (cannot be less than required fee %s)", + if (total_fee < requiredFee) { + errors.push_back(strprintf("Insufficient totalFee (cannot be less than required fee %s)", FormatMoney(requiredFee))); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return; + return Result::INVALID_PARAMETER; } - nNewFee = totalFee; - nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); + new_fee = total_fee; + nNewFeeRate = CFeeRate(total_fee, maxNewTxSize); } else { - nNewFee = GetMinimumFee(maxNewTxSize, coin_control, mempool, ::feeEstimator, nullptr /* FeeCalculation */); - nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); + new_fee = GetMinimumFee(maxNewTxSize, coin_control, mempool, ::feeEstimator, nullptr /* FeeCalculation */); + nNewFeeRate = CFeeRate(new_fee, maxNewTxSize); // New fee rate must be at least old rate + minimum incremental relay rate // walletIncrementalRelayFee.GetFeePerK() should be exact, because it's initialized @@ -177,47 +174,50 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, const CCoin // add 1 satoshi to the result, because it may have been rounded down. if (nNewFeeRate.GetFeePerK() < nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()) { nNewFeeRate = CFeeRate(nOldFeeRate.GetFeePerK() + 1 + walletIncrementalRelayFee.GetFeePerK()); - nNewFee = nNewFeeRate.GetFee(maxNewTxSize); + new_fee = nNewFeeRate.GetFee(maxNewTxSize); } } // Check that in all cases the new fee doesn't violate maxTxFee - if (nNewFee > maxTxFee) { - vErrors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", - FormatMoney(nNewFee), FormatMoney(maxTxFee))); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + if (new_fee > maxTxFee) { + errors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", + FormatMoney(new_fee), FormatMoney(maxTxFee))); + return Result::WALLET_ERROR; } // check that fee rate is higher than mempool's minimum fee // (no point in bumping fee if we know that the new tx won't be accepted to the mempool) // This may occur if the user set TotalFee or paytxfee too low, if fallbackfee is too low, or, perhaps, // in a rare situation where the mempool minimum fee increased significantly since the fee estimation just a - // moment earlier. In this case, we report an error to the user, who may use totalFee to make an adjustment. + // moment earlier. In this case, we report an error to the user, who may use total_fee to make an adjustment. CFeeRate minMempoolFeeRate = mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); if (nNewFeeRate.GetFeePerK() < minMempoolFeeRate.GetFeePerK()) { - vErrors.push_back(strprintf("New fee rate (%s) is less than the minimum fee rate (%s) to get into the mempool. totalFee value should to be at least %s or settxfee value should be at least %s to add transaction.", FormatMoney(nNewFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFeePerK()), FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), FormatMoney(minMempoolFeeRate.GetFeePerK()))); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + errors.push_back(strprintf( + "New fee rate (%s) is lower than the minimum fee rate (%s) to get into the mempool -- " + "the totalFee value should be at least %s or the settxfee value should be at least %s to add transaction", + FormatMoney(nNewFeeRate.GetFeePerK()), + FormatMoney(minMempoolFeeRate.GetFeePerK()), + FormatMoney(minMempoolFeeRate.GetFee(maxNewTxSize)), + FormatMoney(minMempoolFeeRate.GetFeePerK()))); + return Result::WALLET_ERROR; } // Now modify the output to increase the fee. // If the output is not large enough to pay the fee, fail. - CAmount nDelta = nNewFee - nOldFee; + CAmount nDelta = new_fee - old_fee; assert(nDelta > 0); mtx = *wtx.tx; CTxOut* poutput = &(mtx.vout[nOutput]); if (poutput->nValue < nDelta) { - vErrors.push_back("Change output is too small to bump the fee"); - currentResult = BumpFeeResult::WALLET_ERROR; - return; + errors.push_back("Change output is too small to bump the fee"); + return Result::WALLET_ERROR; } // If the output would become dust, discard it (converting the dust to fee) poutput->nValue -= nDelta; if (poutput->nValue <= GetDustThreshold(*poutput, ::dustRelayFee)) { LogPrint(BCLog::RPC, "Bumping fee and discarding dust output\n"); - nNewFee += poutput->nValue; + new_fee += poutput->nValue; mtx.vout.erase(mtx.vout.begin() + nOutput); } @@ -228,36 +228,36 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, const CCoin } } - currentResult = BumpFeeResult::OK; + return Result::OK; } -bool CFeeBumper::signTransaction(CWallet *pWallet) -{ - return pWallet->SignTransaction(mtx); +bool SignTransaction(CWallet* wallet, CMutableTransaction& mtx) { + LOCK2(cs_main, wallet->cs_wallet); + return wallet->SignTransaction(mtx); } -bool CFeeBumper::commit(CWallet *pWallet) +Result CommitTransaction(CWallet* wallet, const uint256& txid, CMutableTransaction&& mtx, std::vector<std::string>& errors, uint256& bumped_txid) { - AssertLockHeld(pWallet->cs_wallet); - if (!vErrors.empty() || currentResult != BumpFeeResult::OK) { - return false; + LOCK2(cs_main, wallet->cs_wallet); + if (!errors.empty()) { + return Result::MISC_ERROR; } - auto it = txid.IsNull() ? pWallet->mapWallet.end() : pWallet->mapWallet.find(txid); - if (it == pWallet->mapWallet.end()) { - vErrors.push_back("Invalid or non-wallet transaction id"); - currentResult = BumpFeeResult::MISC_ERROR; - return false; + auto it = txid.IsNull() ? wallet->mapWallet.end() : wallet->mapWallet.find(txid); + if (it == wallet->mapWallet.end()) { + errors.push_back("Invalid or non-wallet transaction id"); + return Result::MISC_ERROR; } CWalletTx& oldWtx = it->second; // make sure the transaction still has no descendants and hasn't been mined in the meantime - if (!preconditionChecks(pWallet, oldWtx)) { - return false; + Result result = PreconditionChecks(wallet, oldWtx, errors); + if (result != Result::OK) { + return result; } - CWalletTx wtxBumped(pWallet, MakeTransactionRef(std::move(mtx))); + CWalletTx wtxBumped(wallet, MakeTransactionRef(std::move(mtx))); // commit/broadcast the tx - CReserveKey reservekey(pWallet); + CReserveKey reservekey(wallet); wtxBumped.mapValue = oldWtx.mapValue; wtxBumped.mapValue["replaces_txid"] = oldWtx.GetHash().ToString(); wtxBumped.vOrderForm = oldWtx.vOrderForm; @@ -265,27 +265,29 @@ bool CFeeBumper::commit(CWallet *pWallet) wtxBumped.fTimeReceivedIsTxTime = true; wtxBumped.fFromMe = true; CValidationState state; - if (!pWallet->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { + if (!wallet->CommitTransaction(wtxBumped, reservekey, g_connman.get(), state)) { // NOTE: CommitTransaction never returns false, so this should never happen. - vErrors.push_back(strprintf("Error: The transaction was rejected! Reason given: %s", state.GetRejectReason())); - return false; + errors.push_back(strprintf("The transaction was rejected: %s", state.GetRejectReason())); + return Result::WALLET_ERROR; } - bumpedTxid = wtxBumped.GetHash(); + bumped_txid = wtxBumped.GetHash(); if (state.IsInvalid()) { // This can happen if the mempool rejected the transaction. Report // what happened in the "errors" response. - vErrors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); + errors.push_back(strprintf("Error: The transaction was rejected: %s", FormatStateMessage(state))); } // mark the original tx as bumped - if (!pWallet->MarkReplaced(oldWtx.GetHash(), wtxBumped.GetHash())) { + if (!wallet->MarkReplaced(oldWtx.GetHash(), wtxBumped.GetHash())) { // TODO: see if JSON-RPC has a standard way of returning a response // along with an exception. It would be good to return information about // wtxBumped to the caller even if marking the original transaction // replaced does not succeed for some reason. - vErrors.push_back("Error: Created new bumpfee transaction but could not mark the original transaction as replaced."); + errors.push_back("Created new bumpfee transaction but could not mark the original transaction as replaced"); } - return true; + return Result::OK; } +} // namespace feebumper + diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index 3d64e53c15..8eec30440c 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -13,7 +13,9 @@ class uint256; class CCoinControl; enum class FeeEstimateMode; -enum class BumpFeeResult +namespace feebumper { + +enum class Result { OK, INVALID_ADDRESS_OR_KEY, @@ -23,39 +25,34 @@ enum class BumpFeeResult MISC_ERROR, }; -class CFeeBumper -{ -public: - CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, const CCoinControl& coin_control, CAmount totalFee); - BumpFeeResult getResult() const { return currentResult; } - const std::vector<std::string>& getErrors() const { return vErrors; } - CAmount getOldFee() const { return nOldFee; } - CAmount getNewFee() const { return nNewFee; } - uint256 getBumpedTxId() const { return bumpedTxid; } - - /* signs the new transaction, - * returns false if the tx couldn't be found or if it was - * impossible to create the signature(s) - */ - bool signTransaction(CWallet *pWallet); - - /* commits the fee bump, - * returns true, in case of CWallet::CommitTransaction was successful - * but, eventually sets vErrors if the tx could not be added to the mempool (will try later) - * or if the old transaction could not be marked as replaced - */ - bool commit(CWallet *pWalletNonConst); - -private: - bool preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx); - - const uint256 txid; - uint256 bumpedTxid; - CMutableTransaction mtx; - std::vector<std::string> vErrors; - BumpFeeResult currentResult; - CAmount nOldFee; - CAmount nNewFee; -}; +//! Return whether transaction can be bumped. +bool TransactionCanBeBumped(CWallet* wallet, const uint256& txid); + +//! Create bumpfee transaction. +Result CreateTransaction(const CWallet* wallet, + const uint256& txid, + const CCoinControl& coin_control, + CAmount total_fee, + std::vector<std::string>& errors, + CAmount& old_fee, + CAmount& new_fee, + CMutableTransaction& mtx); + +//! Sign the new transaction, +//! @return false if the tx couldn't be found or if it was +//! impossible to create the signature(s) +bool SignTransaction(CWallet* wallet, CMutableTransaction& mtx); + +//! Commit the bumpfee transaction. +//! @return success in case of CWallet::CommitTransaction was successful, +//! but sets errors if the tx could not be added to the mempool (will try later) +//! or if the old transaction could not be marked as replaced. +Result CommitTransaction(CWallet* wallet, + const uint256& txid, + CMutableTransaction&& mtx, + std::vector<std::string>& errors, + uint256& bumped_txid); + +} // namespace feebumper #endif // BITCOIN_WALLET_FEEBUMPER_H diff --git a/src/wallet/fees.cpp b/src/wallet/fees.cpp index 76eeeeda05..73985dcf25 100644 --- a/src/wallet/fees.cpp +++ b/src/wallet/fees.cpp @@ -3,14 +3,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/fees.h" +#include <wallet/fees.h> -#include "policy/policy.h" -#include "txmempool.h" -#include "util.h" -#include "validation.h" -#include "wallet/coincontrol.h" -#include "wallet/wallet.h" +#include <policy/policy.h> +#include <txmempool.h> +#include <util.h> +#include <validation.h> +#include <wallet/coincontrol.h> +#include <wallet/wallet.h> CAmount GetRequiredFee(unsigned int nTxBytes) diff --git a/src/wallet/fees.h b/src/wallet/fees.h index 7b8a7dc868..225aff08ad 100644 --- a/src/wallet/fees.h +++ b/src/wallet/fees.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_WALLET_FEES_H #define BITCOIN_WALLET_FEES_H -#include "amount.h" +#include <amount.h> class CBlockPolicyEstimator; class CCoinControl; diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index c984df1df8..0f453f154f 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -3,18 +3,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/init.h" +#include <wallet/init.h> -#include "net.h" -#include "util.h" -#include "utilmoneystr.h" -#include "validation.h" -#include "wallet/wallet.h" -#include "wallet/rpcwallet.h" +#include <net.h> +#include <util.h> +#include <utilmoneystr.h> +#include <validation.h> +#include <wallet/rpcwallet.h> +#include <wallet/wallet.h> +#include <wallet/walletutil.h> std::string GetWalletHelpString(bool showDebug) { std::string strUsage = HelpMessageGroup(_("Wallet options:")); + strUsage += HelpMessageOpt("-addresstype", strprintf(_("What type of addresses to use (\"legacy\", \"p2sh-segwit\", or \"bech32\", default: \"%s\")"), FormatOutputType(OUTPUT_TYPE_DEFAULT))); + strUsage += HelpMessageOpt("-changetype", _("What type of change to use (\"legacy\", \"p2sh-segwit\", or \"bech32\", default is same as -addresstype)")); strUsage += HelpMessageOpt("-disablewallet", _("Do not load the wallet and disable wallet RPC calls")); strUsage += HelpMessageOpt("-keypool=<n>", strprintf(_("Set key pool size to <n> (default: %u)"), DEFAULT_KEYPOOL_SIZE)); strUsage += HelpMessageOpt("-fallbackfee=<amt>", strprintf(_("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data (default: %s)"), @@ -30,10 +33,11 @@ std::string GetWalletHelpString(bool showDebug) strUsage += HelpMessageOpt("-salvagewallet", _("Attempt to recover private keys from a corrupt wallet on startup")); strUsage += HelpMessageOpt("-spendzeroconfchange", strprintf(_("Spend unconfirmed change when sending transactions (default: %u)"), DEFAULT_SPEND_ZEROCONF_CHANGE)); strUsage += HelpMessageOpt("-txconfirmtarget=<n>", strprintf(_("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)"), DEFAULT_TX_CONFIRM_TARGET)); - strUsage += HelpMessageOpt("-walletrbf", strprintf(_("Send transactions with full-RBF opt-in enabled (default: %u)"), DEFAULT_WALLET_RBF)); + strUsage += HelpMessageOpt("-walletrbf", strprintf(_("Send transactions with full-RBF opt-in enabled (RPC only, default: %u)"), DEFAULT_WALLET_RBF)); strUsage += HelpMessageOpt("-upgradewallet", _("Upgrade wallet to latest format on startup")); strUsage += HelpMessageOpt("-wallet=<file>", _("Specify wallet file (within data directory)") + " " + strprintf(_("(default: %s)"), DEFAULT_WALLET_DAT)); strUsage += HelpMessageOpt("-walletbroadcast", _("Make the wallet broadcast transactions") + " " + strprintf(_("(default: %u)"), DEFAULT_WALLETBROADCAST)); + strUsage += HelpMessageOpt("-walletdir=<dir>", _("Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)")); strUsage += HelpMessageOpt("-walletnotify=<cmd>", _("Execute command when a wallet transaction changes (%s in cmd is replaced by TxID)")); strUsage += HelpMessageOpt("-zapwallettxes=<mode>", _("Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup") + " " + _("(1 = keep tx meta data e.g. account owner and payment request information, 2 = drop tx meta data)")); @@ -53,11 +57,16 @@ std::string GetWalletHelpString(bool showDebug) bool WalletParameterInteraction() { - gArgs.SoftSetArg("-wallet", DEFAULT_WALLET_DAT); - const bool is_multiwallet = gArgs.GetArgs("-wallet").size() > 1; + if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { + for (const std::string& wallet : gArgs.GetArgs("-wallet")) { + LogPrintf("%s: parameter interaction: -disablewallet -> ignoring -wallet=%s\n", __func__, wallet); + } - if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) return true; + } + + gArgs.SoftSetArg("-wallet", DEFAULT_WALLET_DAT); + const bool is_multiwallet = gArgs.GetArgs("-wallet").size() > 1; if (gArgs.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY) && gArgs.SoftSetBoolArg("-walletbroadcast", false)) { LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); @@ -168,20 +177,42 @@ bool WalletParameterInteraction() bSpendZeroConfChange = gArgs.GetBoolArg("-spendzeroconfchange", DEFAULT_SPEND_ZEROCONF_CHANGE); fWalletRbf = gArgs.GetBoolArg("-walletrbf", DEFAULT_WALLET_RBF); + g_address_type = ParseOutputType(gArgs.GetArg("-addresstype", "")); + if (g_address_type == OUTPUT_TYPE_NONE) { + return InitError(strprintf(_("Unknown address type '%s'"), gArgs.GetArg("-addresstype", ""))); + } + + g_change_type = ParseOutputType(gArgs.GetArg("-changetype", ""), g_address_type); + if (g_change_type == OUTPUT_TYPE_NONE) { + return InitError(strprintf(_("Unknown change type '%s'"), gArgs.GetArg("-changetype", ""))); + } + return true; } void RegisterWalletRPC(CRPCTable &t) { - if (gArgs.GetBoolArg("-disablewallet", false)) return; + if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { + return; + } RegisterWalletRPCCommands(t); } bool VerifyWallets() { - if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) + if (gArgs.GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { return true; + } + + if (gArgs.IsArgSet("-walletdir") && !fs::is_directory(GetWalletDir())) { + if (fs::exists(fs::system_complete(gArgs.GetArg("-walletdir", "")))) { + return InitError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), gArgs.GetArg("-walletdir", "").c_str())); + } + return InitError(strprintf(_("Specified -walletdir \"%s\" does not exist"), gArgs.GetArg("-walletdir", "").c_str())); + } + + LogPrintf("Using wallet directory %s\n", GetWalletDir().string()); uiInterface.InitMessage(_("Verifying wallet(s)...")); @@ -197,7 +228,7 @@ bool VerifyWallets() return InitError(strprintf(_("Error loading wallet %s. Invalid characters in -wallet filename."), walletFile)); } - fs::path wallet_path = fs::absolute(walletFile, GetDataDir()); + fs::path wallet_path = fs::absolute(walletFile, GetWalletDir()); if (fs::exists(wallet_path) && (!fs::is_regular_file(wallet_path) || fs::is_symlink(wallet_path))) { return InitError(strprintf(_("Error loading wallet %s. -wallet filename must be a regular file."), walletFile)); @@ -208,7 +239,7 @@ bool VerifyWallets() } std::string strError; - if (!CWalletDB::VerifyEnvironment(walletFile, GetDataDir().string(), strError)) { + if (!CWalletDB::VerifyEnvironment(walletFile, GetWalletDir().string(), strError)) { return InitError(strError); } @@ -222,7 +253,7 @@ bool VerifyWallets() } std::string strWarning; - bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetDataDir().string(), strWarning, strError); + bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetWalletDir().string(), strWarning, strError); if (!strWarning.empty()) { InitWarning(strWarning); } diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index d6ea2a9db7..645e776b68 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -1,23 +1,23 @@ -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "base58.h" -#include "chain.h" -#include "rpc/safemode.h" -#include "rpc/server.h" -#include "init.h" -#include "validation.h" -#include "script/script.h" -#include "script/standard.h" -#include "sync.h" -#include "util.h" -#include "utiltime.h" -#include "wallet.h" -#include "merkleblock.h" -#include "core_io.h" - -#include "rpcwallet.h" +#include <base58.h> +#include <chain.h> +#include <rpc/safemode.h> +#include <rpc/server.h> +#include <wallet/init.h> +#include <validation.h> +#include <script/script.h> +#include <script/standard.h> +#include <sync.h> +#include <util.h> +#include <utiltime.h> +#include <wallet/wallet.h> +#include <merkleblock.h> +#include <core_io.h> + +#include <wallet/rpcwallet.h> #include <fstream> #include <stdint.h> @@ -81,7 +81,7 @@ UniValue importprivkey(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) throw std::runtime_error( "importprivkey \"privkey\" ( \"label\" ) ( rescan )\n" - "\nAdds a private key (as returned by dumpprivkey) to your wallet.\n" + "\nAdds a private key (as returned by dumpprivkey) to your wallet. Requires a new wallet backup.\n" "\nArguments:\n" "1. \"privkey\" (string, required) The private key (see dumpprivkey)\n" "2. \"label\" (string, optional, default=\"\") An optional label\n" @@ -131,7 +131,11 @@ UniValue importprivkey(const JSONRPCRequest& request) CKeyID vchAddress = pubkey.GetID(); { pwallet->MarkDirty(); - pwallet->SetAddressBook(vchAddress, strLabel, "receive"); + + // We don't know which corresponding address will be used; label them all + for (const auto& dest : GetAllDestinationsForKey(pubkey)) { + pwallet->SetAddressBook(dest, strLabel, "receive"); + } // Don't throw error in case a key is already there if (pwallet->HaveKey(vchAddress)) { @@ -143,6 +147,7 @@ UniValue importprivkey(const JSONRPCRequest& request) if (!pwallet->AddKeyPubKey(key, pubkey)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); } + pwallet->LearnAllRelatedScripts(pubkey); // whenever a key is imported, we need to scan the whole chain pwallet->UpdateTimeFirstKey(1); @@ -165,7 +170,7 @@ UniValue abortrescan(const JSONRPCRequest& request) if (request.fHelp || request.params.size() > 0) throw std::runtime_error( "abortrescan\n" - "\nStops current wallet rescan triggered e.g. by an importprivkey call.\n" + "\nStops current wallet rescan triggered by an RPC call, e.g. by an importprivkey call.\n" "\nExamples:\n" "\nImport a private key\n" + HelpExampleCli("importprivkey", "\"mykey\"") + @@ -226,7 +231,7 @@ UniValue importaddress(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) throw std::runtime_error( "importaddress \"address\" ( \"label\" rescan p2sh )\n" - "\nAdds a script (in hex) or address that can be watched as if it were in your wallet but cannot be used to spend.\n" + "\nAdds a script (in hex) or address that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "\nArguments:\n" "1. \"script\" (string, required) The hex-encoded script (or address)\n" "2. \"label\" (string, optional, default=\"\") An optional label\n" @@ -340,7 +345,7 @@ UniValue importprunedfunds(const JSONRPCRequest& request) LOCK2(cs_main, pwallet->cs_wallet); - if (pwallet->IsMine(wtx)) { + if (pwallet->IsMine(*wtx.tx)) { pwallet->AddToWallet(wtx, false); return NullUniValue; } @@ -396,7 +401,7 @@ UniValue importpubkey(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) throw std::runtime_error( "importpubkey \"pubkey\" ( \"label\" rescan )\n" - "\nAdds a public key (in hex) that can be watched as if it were in your wallet but cannot be used to spend.\n" + "\nAdds a public key (in hex) that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "\nArguments:\n" "1. \"pubkey\" (string, required) The hex-encoded public key\n" "2. \"label\" (string, optional, default=\"\") An optional label\n" @@ -433,8 +438,11 @@ UniValue importpubkey(const JSONRPCRequest& request) LOCK2(cs_main, pwallet->cs_wallet); - ImportAddress(pwallet, pubKey.GetID(), strLabel); + for (const auto& dest : GetAllDestinationsForKey(pubKey)) { + ImportAddress(pwallet, dest, strLabel); + } ImportScript(pwallet, GetScriptForRawPubKey(pubKey), strLabel, false); + pwallet->LearnAllRelatedScripts(pubKey); if (fRescan) { @@ -456,7 +464,7 @@ UniValue importwallet(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( "importwallet \"filename\"\n" - "\nImports keys from a wallet dump file (see dumpwallet).\n" + "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n" "\nArguments:\n" "1. \"filename\" (string, required) The wallet file\n" "\nExamples:\n" @@ -500,40 +508,57 @@ UniValue importwallet(const JSONRPCRequest& request) if (vstr.size() < 2) continue; CBitcoinSecret vchSecret; - if (!vchSecret.SetString(vstr[0])) - continue; - CKey key = vchSecret.GetKey(); - CPubKey pubkey = key.GetPubKey(); - assert(key.VerifyPubKey(pubkey)); - CKeyID keyid = pubkey.GetID(); - if (pwallet->HaveKey(keyid)) { - LogPrintf("Skipping import of %s (key already present)\n", EncodeDestination(keyid)); - continue; - } - int64_t nTime = DecodeDumpTime(vstr[1]); - std::string strLabel; - bool fLabel = true; - for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) { - if (boost::algorithm::starts_with(vstr[nStr], "#")) - break; - if (vstr[nStr] == "change=1") - fLabel = false; - if (vstr[nStr] == "reserve=1") - fLabel = false; - if (boost::algorithm::starts_with(vstr[nStr], "label=")) { - strLabel = DecodeDumpString(vstr[nStr].substr(6)); - fLabel = true; + if (vchSecret.SetString(vstr[0])) { + CKey key = vchSecret.GetKey(); + CPubKey pubkey = key.GetPubKey(); + assert(key.VerifyPubKey(pubkey)); + CKeyID keyid = pubkey.GetID(); + if (pwallet->HaveKey(keyid)) { + LogPrintf("Skipping import of %s (key already present)\n", EncodeDestination(keyid)); + continue; } + int64_t nTime = DecodeDumpTime(vstr[1]); + std::string strLabel; + bool fLabel = true; + for (unsigned int nStr = 2; nStr < vstr.size(); nStr++) { + if (boost::algorithm::starts_with(vstr[nStr], "#")) + break; + if (vstr[nStr] == "change=1") + fLabel = false; + if (vstr[nStr] == "reserve=1") + fLabel = false; + if (boost::algorithm::starts_with(vstr[nStr], "label=")) { + strLabel = DecodeDumpString(vstr[nStr].substr(6)); + fLabel = true; + } + } + LogPrintf("Importing %s...\n", EncodeDestination(keyid)); + if (!pwallet->AddKeyPubKey(key, pubkey)) { + fGood = false; + continue; + } + pwallet->mapKeyMetadata[keyid].nCreateTime = nTime; + if (fLabel) + pwallet->SetAddressBook(keyid, strLabel, "receive"); + nTimeBegin = std::min(nTimeBegin, nTime); + } else if(IsHex(vstr[0])) { + std::vector<unsigned char> vData(ParseHex(vstr[0])); + CScript script = CScript(vData.begin(), vData.end()); + if (pwallet->HaveCScript(script)) { + LogPrintf("Skipping import of %s (script already present)\n", vstr[0]); + continue; + } + if(!pwallet->AddCScript(script)) { + LogPrintf("Error importing script %s\n", vstr[0]); + fGood = false; + continue; + } + int64_t birth_time = DecodeDumpTime(vstr[1]); + if (birth_time > 0) { + pwallet->m_script_metadata[CScriptID(script)].nCreateTime = birth_time; + nTimeBegin = std::min(nTimeBegin, birth_time); + } } - LogPrintf("Importing %s...\n", EncodeDestination(keyid)); - if (!pwallet->AddKeyPubKey(key, pubkey)) { - fGood = false; - continue; - } - pwallet->mapKeyMetadata[keyid].nCreateTime = nTime; - if (fLabel) - pwallet->SetAddressBook(keyid, strLabel, "receive"); - nTimeBegin = std::min(nTimeBegin, nTime); } file.close(); pwallet->ShowProgress("", 100); // hide progress dialog in GUI @@ -542,7 +567,7 @@ UniValue importwallet(const JSONRPCRequest& request) pwallet->MarkDirty(); if (!fGood) - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys to wallet"); + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys/scripts to wallet"); return NullUniValue; } @@ -578,12 +603,12 @@ UniValue dumpprivkey(const JSONRPCRequest& request) if (!IsValidDestination(dest)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); } - const CKeyID *keyID = boost::get<CKeyID>(&dest); - if (!keyID) { + auto keyid = GetKeyForDestination(*pwallet, dest); + if (keyid.IsNull()) { throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to a key"); } CKey vchSecret; - if (!pwallet->GetKey(*keyID, vchSecret)) { + if (!pwallet->GetKey(keyid, vchSecret)) { throw JSONRPCError(RPC_WALLET_ERROR, "Private key for address " + strAddress + " is not known"); } return CBitcoinSecret(vchSecret).ToString(); @@ -601,6 +626,9 @@ UniValue dumpwallet(const JSONRPCRequest& request) throw std::runtime_error( "dumpwallet \"filename\"\n" "\nDumps all wallet keys in a human-readable format to a server-side file. This does not allow overwriting existing files.\n" + "Imported scripts are included in the dumpfile, but corresponding BIP173 addresses, etc. may not be added automatically by importwallet.\n" + "Note that if your wallet contains keys which are not derived from your HD seed (e.g. imported keys), these are not covered by\n" + "only backing up the seed itself, and must be backed up too (e.g. ensure you back up the whole dumpfile).\n" "\nArguments:\n" "1. \"filename\" (string, required) The filename with path (either absolute or relative to bitcoind)\n" "\nResult:\n" @@ -637,6 +665,9 @@ UniValue dumpwallet(const JSONRPCRequest& request) const std::map<CKeyID, int64_t>& mapKeyPool = pwallet->GetAllReserveKeys(); pwallet->GetKeyBirthTimes(mapKeyBirth); + std::set<CScriptID> scripts = pwallet->GetCScripts(); + // TODO: include scripts in GetKeyBirthTimes() output instead of separate + // sort time/key pairs std::vector<std::pair<int64_t, CKeyID> > vKeyBirth; for (const auto& entry : mapKeyBirth) { @@ -691,6 +722,21 @@ UniValue dumpwallet(const JSONRPCRequest& request) } } file << "\n"; + for (const CScriptID &scriptid : scripts) { + CScript script; + std::string create_time = "0"; + std::string address = EncodeDestination(scriptid); + // get birth times for scripts with metadata + auto it = pwallet->m_script_metadata.find(scriptid); + if (it != pwallet->m_script_metadata.end()) { + create_time = EncodeDumpTime(it->second.nCreateTime); + } + if(pwallet->GetCScript(scriptid, script)) { + file << strprintf("%s %s script=1", HexStr(script.begin(), script.end()), create_time); + file << strprintf(" # addr=%s\n", address); + } + } + file << "\n"; file << "# End of dump\n"; file.close(); @@ -718,8 +764,8 @@ UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, const int6 const std::string& strRedeemScript = data.exists("redeemscript") ? data["redeemscript"].get_str() : ""; const UniValue& pubKeys = data.exists("pubkeys") ? data["pubkeys"].get_array() : UniValue(); const UniValue& keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); - const bool& internal = data.exists("internal") ? data["internal"].get_bool() : false; - const bool& watchOnly = data.exists("watchonly") ? data["watchonly"].get_bool() : false; + const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; + const bool watchOnly = data.exists("watchonly") ? data["watchonly"].get_bool() : false; const std::string& label = data.exists("label") && !internal ? data["label"].get_str() : ""; bool isScript = scriptPubKey.getType() == UniValue::VSTR; @@ -961,7 +1007,7 @@ UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, const int6 pwallet->SetAddressBook(vchAddress, label, "receive"); if (pwallet->HaveKey(vchAddress)) { - return false; + throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); } pwallet->mapKeyMetadata[vchAddress].nCreateTime = timestamp; @@ -1039,7 +1085,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) if (mainRequest.fHelp || mainRequest.params.size() < 1 || mainRequest.params.size() > 2) throw std::runtime_error( "importmulti \"requests\" ( \"options\" )\n\n" - "Import addresses/scripts (with private or public keys, redeem script (P2SH)), rescanning all addresses in one-shot-only (rescan can be disabled via options).\n\n" + "Import addresses/scripts (with private or public keys, redeem script (P2SH)), rescanning all addresses in one-shot-only (rescan can be disabled via options). Requires a new wallet backup.\n\n" "Arguments:\n" "1. requests (array, required) Data to be imported\n" " [ (array of json objects)\n" diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 5d98498a4b..e307623fd5 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1,31 +1,32 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "amount.h" -#include "base58.h" -#include "chain.h" -#include "consensus/validation.h" -#include "core_io.h" -#include "httpserver.h" -#include "validation.h" -#include "net.h" -#include "policy/feerate.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "rpc/mining.h" -#include "rpc/safemode.h" -#include "rpc/server.h" -#include "script/sign.h" -#include "timedata.h" -#include "util.h" -#include "utilmoneystr.h" -#include "wallet/coincontrol.h" -#include "wallet/feebumper.h" -#include "wallet/wallet.h" -#include "wallet/walletdb.h" +#include <amount.h> +#include <base58.h> +#include <chain.h> +#include <consensus/validation.h> +#include <core_io.h> +#include <httpserver.h> +#include <validation.h> +#include <net.h> +#include <policy/feerate.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <rpc/mining.h> +#include <rpc/safemode.h> +#include <rpc/server.h> +#include <script/sign.h> +#include <timedata.h> +#include <util.h> +#include <utilmoneystr.h> +#include <wallet/coincontrol.h> +#include <wallet/feebumper.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> +#include <wallet/walletutil.h> #include <init.h> // For StartShutdown @@ -108,7 +109,7 @@ void WalletTxToJSON(const CWalletTx& wtx, UniValue& entry) std::string rbfStatus = "no"; if (confirms <= 0) { LOCK(mempool.cs); - RBFTransactionState rbfState = IsRBFOptIn(wtx, mempool); + RBFTransactionState rbfState = IsRBFOptIn(*wtx.tx, mempool); if (rbfState == RBF_TRANSACTIONSTATE_UNKNOWN) rbfStatus = "unknown"; else if (rbfState == RBF_TRANSACTIONSTATE_REPLACEABLE_BIP125) @@ -135,14 +136,15 @@ UniValue getnewaddress(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() > 1) + if (request.fHelp || request.params.size() > 2) throw std::runtime_error( - "getnewaddress ( \"account\" )\n" + "getnewaddress ( \"account\" \"address_type\" )\n" "\nReturns a new Bitcoin address for receiving payments.\n" "If 'account' is specified (DEPRECATED), it is added to the address book \n" "so payments received with the address will be credited to 'account'.\n" "\nArguments:\n" "1. \"account\" (string, optional) DEPRECATED. The account name for the address to be linked to. If not provided, the default account \"\" is used. It can also be set to the empty string \"\" to represent the default account. The account does not need to exist, it will be created if there is no account by the given name.\n" + "2. \"address_type\" (string, optional) The address type to use. Options are \"legacy\", \"p2sh\", and \"bech32\". Default is set by -addresstype.\n" "\nResult:\n" "\"address\" (string) The new bitcoin address\n" "\nExamples:\n" @@ -157,6 +159,14 @@ UniValue getnewaddress(const JSONRPCRequest& request) if (!request.params[0].isNull()) strAccount = AccountFromValue(request.params[0]); + OutputType output_type = g_address_type; + if (!request.params[1].isNull()) { + output_type = ParseOutputType(request.params[1].get_str(), g_address_type); + if (output_type == OUTPUT_TYPE_NONE) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[1].get_str())); + } + } + if (!pwallet->IsLocked()) { pwallet->TopUpKeyPool(); } @@ -166,22 +176,23 @@ UniValue getnewaddress(const JSONRPCRequest& request) if (!pwallet->GetKeyFromPool(newKey)) { throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); } - CKeyID keyID = newKey.GetID(); + pwallet->LearnRelatedScripts(newKey, output_type); + CTxDestination dest = GetDestinationForKey(newKey, output_type); - pwallet->SetAddressBook(keyID, strAccount, "receive"); + pwallet->SetAddressBook(dest, strAccount, "receive"); - return EncodeDestination(keyID); + return EncodeDestination(dest); } -CTxDestination GetAccountAddress(CWallet* const pwallet, std::string strAccount, bool bForceNew=false) +CTxDestination GetAccountDestination(CWallet* const pwallet, std::string strAccount, bool bForceNew=false) { - CPubKey pubKey; - if (!pwallet->GetAccountPubkey(pubKey, strAccount, bForceNew)) { + CTxDestination dest; + if (!pwallet->GetAccountDestination(dest, strAccount, bForceNew)) { throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); } - return pubKey.GetID(); + return dest; } UniValue getaccountaddress(const JSONRPCRequest& request) @@ -213,7 +224,7 @@ UniValue getaccountaddress(const JSONRPCRequest& request) UniValue ret(UniValue::VSTR); - ret = EncodeDestination(GetAccountAddress(pwallet, strAccount)); + ret = EncodeDestination(GetAccountDestination(pwallet, strAccount)); return ret; } @@ -225,11 +236,13 @@ UniValue getrawchangeaddress(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() > 0) + if (request.fHelp || request.params.size() > 1) throw std::runtime_error( - "getrawchangeaddress\n" + "getrawchangeaddress ( \"address_type\" )\n" "\nReturns a new Bitcoin address, for receiving change.\n" "This is for use with raw transactions, NOT normal use.\n" + "\nArguments:\n" + "1. \"address_type\" (string, optional) The address type to use. Options are \"legacy\", \"p2sh\", and \"bech32\". Default is set by -changetype.\n" "\nResult:\n" "\"address\" (string) The address\n" "\nExamples:\n" @@ -243,6 +256,14 @@ UniValue getrawchangeaddress(const JSONRPCRequest& request) pwallet->TopUpKeyPool(); } + OutputType output_type = g_change_type; + if (!request.params[0].isNull()) { + output_type = ParseOutputType(request.params[0].get_str(), g_change_type); + if (output_type == OUTPUT_TYPE_NONE) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[0].get_str())); + } + } + CReserveKey reservekey(pwallet); CPubKey vchPubKey; if (!reservekey.GetReservedKey(vchPubKey, true)) @@ -250,9 +271,10 @@ UniValue getrawchangeaddress(const JSONRPCRequest& request) reservekey.KeepKey(); - CKeyID keyID = vchPubKey.GetID(); + pwallet->LearnRelatedScripts(vchPubKey, output_type); + CTxDestination dest = GetDestinationForKey(vchPubKey, output_type); - return EncodeDestination(keyID); + return EncodeDestination(dest); } @@ -291,8 +313,8 @@ UniValue setaccount(const JSONRPCRequest& request) // Detect when changing the account of an address that is the 'unused current key' of another account: if (pwallet->mapAddressBook.count(dest)) { std::string strOldAccount = pwallet->mapAddressBook[dest].name; - if (dest == GetAccountAddress(pwallet, strOldAccount)) { - GetAccountAddress(pwallet, strOldAccount, true); + if (dest == GetAccountDestination(pwallet, strOldAccount)) { + GetAccountDestination(pwallet, strOldAccount, true); } } pwallet->SetAddressBook(dest, strAccount, "receive"); @@ -455,6 +477,11 @@ UniValue sendtoaddress(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); CTxDestination dest = DecodeDestination(request.params[0].get_str()); @@ -533,6 +560,11 @@ UniValue listaddressgroupings(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); UniValue jsonGroupings(UniValue::VARR); @@ -645,6 +677,11 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); // Bitcoin address @@ -654,7 +691,7 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) } CScript scriptPubKey = GetScriptForDestination(dest); if (!IsMine(*pwallet, scriptPubKey)) { - return ValueFromAmount(0); + throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); } // Minimum confirmations @@ -707,6 +744,11 @@ UniValue getreceivedbyaccount(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); // Minimum confirmations @@ -750,6 +792,8 @@ UniValue getbalance(const JSONRPCRequest& request) throw std::runtime_error( "getbalance ( \"account\" minconf include_watchonly )\n" "\nIf account is not specified, returns the server's total available balance.\n" + "The available balance is what the wallet considers currently spendable, and is\n" + "thus affected by options which limit spendability such as -spendzeroconfchange.\n" "If account is specified (DEPRECATED), returns the balance in the account.\n" "Note that the account \"\" is not the same as leaving the parameter out.\n" "The server total may be different to the balance in the default \"\" account.\n" @@ -780,6 +824,11 @@ UniValue getbalance(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); const UniValue& account_value = request.params[0]; @@ -825,6 +874,11 @@ UniValue getunconfirmedbalance(const JSONRPCRequest &request) "Returns the server's total unconfirmed balance\n"); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); return ValueFromAmount(pwallet->GetUnconfirmedBalance()); @@ -919,6 +973,11 @@ UniValue sendfrom(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); std::string strAccount = AccountFromValue(request.params[0]); @@ -1000,10 +1059,15 @@ UniValue sendmany(const JSONRPCRequest& request) "\nSend two amounts to two different addresses, subtract fee from amount:\n" + HelpExampleCli("sendmany", "\"\" \"{\\\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\\\":0.01,\\\"1353tsE8YMTA4EuV7dgUXGjNFf9KpVvKHz\\\":0.02}\" 1 \"\" \"[\\\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\\\",\\\"1353tsE8YMTA4EuV7dgUXGjNFf9KpVvKHz\\\"]\"") + "\nAs a json rpc call\n" - + HelpExampleRpc("sendmany", "\"\", \"{\\\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\\\":0.01,\\\"1353tsE8YMTA4EuV7dgUXGjNFf9KpVvKHz\\\":0.02}\", 6, \"testing\"") + + HelpExampleRpc("sendmany", "\"\", {\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\":0.01,\"1353tsE8YMTA4EuV7dgUXGjNFf9KpVvKHz\":0.02}, 6, \"testing\"") ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); if (pwallet->GetBroadcastTransactions() && !g_connman) { @@ -1110,8 +1174,10 @@ UniValue addmultisigaddress(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 2 || request.params.size() > 3) { std::string msg = "addmultisigaddress nrequired [\"key\",...] ( \"account\" )\n" - "\nAdd a nrequired-to-sign multisignature address to the wallet.\n" + "\nAdd a nrequired-to-sign multisignature address to the wallet. Requires a new wallet backup.\n" "Each key is a Bitcoin address or hex-encoded public key.\n" + "This functionality is only intended for use with non-watchonly addresses.\n" + "See `importaddress` for watchonly p2sh address support.\n" "If 'account' is specified (DEPRECATED), assign address to that account.\n" "\nArguments:\n" @@ -1143,11 +1209,12 @@ UniValue addmultisigaddress(const JSONRPCRequest& request) // Construct using pay-to-script-hash: CScript inner = _createmultisig_redeemScript(pwallet, request.params); - CScriptID innerID(inner); pwallet->AddCScript(inner); - pwallet->SetAddressBook(innerID, strAccount, "send"); - return EncodeDestination(innerID); + CTxDestination dest = pwallet->AddAndGetDestinationForScript(inner, g_address_type); + + pwallet->SetAddressBook(dest, strAccount, "send"); + return EncodeDestination(dest); } class Witnessifier : public boost::static_visitor<bool> @@ -1163,12 +1230,7 @@ public: if (pwallet) { CScript basescript = GetScriptForDestination(keyID); CScript witscript = GetScriptForWitness(basescript); - SignatureData sigs; - // This check is to make sure that the script we created can actually be solved for and signed by us - // if we were to have the private keys. This is just to make sure that the script is valid and that, - // if found in a transaction, we would still accept and relay that transaction. - if (!ProduceSignature(DummySignatureCreator(pwallet), witscript, sigs) || - !VerifyScript(sigs.scriptSig, witscript, &sigs.scriptWitness, MANDATORY_SCRIPT_VERIFY_FLAGS | SCRIPT_VERIFY_WITNESS_PUBKEYTYPE, DummySignatureCreator(pwallet).Checker())) { + if (!IsSolvable(*pwallet, witscript)) { return false; } return ExtractDestination(witscript, result); @@ -1187,12 +1249,7 @@ public: return true; } CScript witscript = GetScriptForWitness(subscript); - SignatureData sigs; - // This check is to make sure that the script we created can actually be solved for and signed by us - // if we were to have the private keys. This is just to make sure that the script is valid and that, - // if found in a transaction, we would still accept and relay that transaction. - if (!ProduceSignature(DummySignatureCreator(pwallet), witscript, sigs) || - !VerifyScript(sigs.scriptSig, witscript, &sigs.scriptWitness, MANDATORY_SCRIPT_VERIFY_FLAGS | SCRIPT_VERIFY_WITNESS_PUBKEYTYPE, DummySignatureCreator(pwallet).Checker())) { + if (!IsSolvable(*pwallet, witscript)) { return false; } return ExtractDestination(witscript, result); @@ -1228,7 +1285,7 @@ UniValue addwitnessaddress(const JSONRPCRequest& request) if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { std::string msg = "addwitnessaddress \"address\" ( p2sh )\n" - "\nAdd a witness address for a script (with pubkey or redeemscript known).\n" + "\nAdd a witness address for a script (with pubkey or redeemscript known). Requires a new wallet backup.\n" "It returns the witness script.\n" "\nArguments:\n" @@ -1276,7 +1333,7 @@ UniValue addwitnessaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Cannot convert between witness address types"); } } else { - pwallet->AddCScript(witprogram); + pwallet->AddCScript(witprogram); // Implicit for single-key now, but necessary for multisig and for compatibility with older software pwallet->SetAddressBook(w.result, "", "receive"); } @@ -1398,14 +1455,14 @@ UniValue ListReceived(CWallet * const pwallet, const UniValue& params, bool fByA if (fByAccounts) { - for (std::map<std::string, tallyitem>::iterator it = mapAccountTally.begin(); it != mapAccountTally.end(); ++it) + for (const auto& entry : mapAccountTally) { - CAmount nAmount = (*it).second.nAmount; - int nConf = (*it).second.nConf; + CAmount nAmount = entry.second.nAmount; + int nConf = entry.second.nConf; UniValue obj(UniValue::VOBJ); - if((*it).second.fIsWatchonly) + if (entry.second.fIsWatchonly) obj.push_back(Pair("involvesWatchonly", true)); - obj.push_back(Pair("account", (*it).first)); + obj.push_back(Pair("account", entry.first)); obj.push_back(Pair("amount", ValueFromAmount(nAmount))); obj.push_back(Pair("confirmations", (nConf == std::numeric_limits<int>::max() ? 0 : nConf))); ret.push_back(obj); @@ -1455,6 +1512,11 @@ UniValue listreceivedbyaddress(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); return ListReceived(pwallet, request.params, false); @@ -1495,6 +1557,11 @@ UniValue listreceivedbyaccount(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); return ListReceived(pwallet, request.params, true); @@ -1683,6 +1750,11 @@ UniValue listtransactions(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); std::string strAccount = "*"; @@ -1777,6 +1849,11 @@ UniValue listaccounts(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); int nMinDepth = 1; @@ -1886,6 +1963,11 @@ UniValue listsinceblock(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); const CBlockIndex* pindex = nullptr; // Block index of the specified block or the common ancestor, if the block provided was in a deactivated chain. @@ -1893,19 +1975,20 @@ UniValue listsinceblock(const JSONRPCRequest& request) int target_confirms = 1; isminefilter filter = ISMINE_SPENDABLE; - if (!request.params[0].isNull()) { + if (!request.params[0].isNull() && !request.params[0].get_str().empty()) { uint256 blockId; blockId.SetHex(request.params[0].get_str()); BlockMap::iterator it = mapBlockIndex.find(blockId); - if (it != mapBlockIndex.end()) { - paltindex = pindex = it->second; - if (chainActive[pindex->nHeight] != pindex) { - // the block being asked for is a part of a deactivated chain; - // we don't want to depend on its perceived height in the block - // chain, we want to instead use the last common ancestor - pindex = chainActive.FindFork(pindex); - } + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + paltindex = pindex = it->second; + if (chainActive[pindex->nHeight] != pindex) { + // the block being asked for is a part of a deactivated chain; + // we don't want to depend on its perceived height in the block + // chain, we want to instead use the last common ancestor + pindex = chainActive.FindFork(pindex); } } @@ -2018,6 +2101,11 @@ UniValue gettransaction(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); uint256 hash; @@ -2050,7 +2138,7 @@ UniValue gettransaction(const JSONRPCRequest& request) ListTransactions(pwallet, wtx, "*", 0, false, details, filter); entry.push_back(Pair("details", details)); - std::string strHex = EncodeHexTx(static_cast<CTransaction>(wtx), RPCSerializationFlags()); + std::string strHex = EncodeHexTx(*wtx.tx, RPCSerializationFlags()); entry.push_back(Pair("hex", strHex)); return entry; @@ -2080,6 +2168,11 @@ UniValue abandontransaction(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); uint256 hash; @@ -2114,6 +2207,10 @@ UniValue backupwallet(const JSONRPCRequest& request) + HelpExampleRpc("backupwallet", "\"backup.dat\"") ); + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); std::string strDest = request.params[0].get_str(); @@ -2179,7 +2276,7 @@ UniValue walletpassphrase(const JSONRPCRequest& request) return NullUniValue; } - if (pwallet->IsCrypted() && (request.fHelp || request.params.size() != 2)) { + if (request.fHelp || request.params.size() != 2) { throw std::runtime_error( "walletpassphrase \"passphrase\" timeout\n" "\nStores the wallet decryption key in memory for 'timeout' seconds.\n" @@ -2243,7 +2340,7 @@ UniValue walletpassphrasechange(const JSONRPCRequest& request) return NullUniValue; } - if (pwallet->IsCrypted() && (request.fHelp || request.params.size() != 2)) { + if (request.fHelp || request.params.size() != 2) { throw std::runtime_error( "walletpassphrasechange \"oldpassphrase\" \"newpassphrase\"\n" "\nChanges the wallet passphrase from 'oldpassphrase' to 'newpassphrase'.\n" @@ -2294,7 +2391,7 @@ UniValue walletlock(const JSONRPCRequest& request) return NullUniValue; } - if (pwallet->IsCrypted() && (request.fHelp || request.params.size() != 0)) { + if (request.fHelp || request.params.size() != 0) { throw std::runtime_error( "walletlock\n" "\nRemoves the wallet encryption key from memory, locking the wallet.\n" @@ -2334,7 +2431,7 @@ UniValue encryptwallet(const JSONRPCRequest& request) return NullUniValue; } - if (!pwallet->IsCrypted() && (request.fHelp || request.params.size() != 1)) { + if (request.fHelp || request.params.size() != 1) { throw std::runtime_error( "encryptwallet \"passphrase\"\n" "\nEncrypts the wallet with 'passphrase'. This is for first time encryption.\n" @@ -2433,6 +2530,10 @@ UniValue lockunspent(const JSONRPCRequest& request) + HelpExampleRpc("lockunspent", "false, \"[{\\\"txid\\\":\\\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\\\",\\\"vout\\\":1}]\"") ); + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); RPCTypeCheckArgument(request.params[0], UniValue::VBOOL); @@ -2447,12 +2548,15 @@ UniValue lockunspent(const JSONRPCRequest& request) RPCTypeCheckArgument(request.params[1], UniValue::VARR); - UniValue outputs = request.params[1].get_array(); - for (unsigned int idx = 0; idx < outputs.size(); idx++) { - const UniValue& output = outputs[idx]; - if (!output.isObject()) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected object"); - const UniValue& o = output.get_obj(); + const UniValue& output_params = request.params[1]; + + // Create and validate the COutPoints first. + + std::vector<COutPoint> outputs; + outputs.reserve(output_params.size()); + + for (unsigned int idx = 0; idx < output_params.size(); idx++) { + const UniValue& o = output_params[idx].get_obj(); RPCTypeCheckObj(o, { @@ -2460,20 +2564,50 @@ UniValue lockunspent(const JSONRPCRequest& request) {"vout", UniValueType(UniValue::VNUM)}, }); - std::string txid = find_value(o, "txid").get_str(); - if (!IsHex(txid)) + const std::string& txid = find_value(o, "txid").get_str(); + if (!IsHex(txid)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected hex txid"); + } - int nOutput = find_value(o, "vout").get_int(); - if (nOutput < 0) + const int nOutput = find_value(o, "vout").get_int(); + if (nOutput < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout must be positive"); + } - COutPoint outpt(uint256S(txid), nOutput); + const COutPoint outpt(uint256S(txid), nOutput); - if (fUnlock) - pwallet->UnlockCoin(outpt); - else - pwallet->LockCoin(outpt); + const auto it = pwallet->mapWallet.find(outpt.hash); + if (it == pwallet->mapWallet.end()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, unknown transaction"); + } + + const CWalletTx& trans = it->second; + + if (outpt.n >= trans.tx->vout.size()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout index out of bounds"); + } + + if (pwallet->IsSpent(outpt.hash, outpt.n)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected unspent output"); + } + + const bool is_locked = pwallet->IsLockedCoin(outpt.hash, outpt.n); + + if (fUnlock && !is_locked) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected locked output"); + } + + if (!fUnlock && is_locked) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, output already locked"); + } + + outputs.push_back(outpt); + } + + // Atomically set (un)locked status for the outputs. + for (const COutPoint& outpt : outputs) { + if (fUnlock) pwallet->UnlockCoin(outpt); + else pwallet->LockCoin(outpt); } return true; @@ -2592,6 +2726,11 @@ UniValue getwalletinfo(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); UniValue obj(UniValue::VOBJ); @@ -2801,9 +2940,12 @@ UniValue listunspent(const JSONRPCRequest& request) nMaximumCount = options["maximumCount"].get_int64(); } + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + UniValue results(UniValue::VARR); std::vector<COutput> vecOutputs; - assert(pwallet != nullptr); LOCK2(cs_main, pwallet->cs_wallet); pwallet->AvailableCoins(vecOutputs, !include_unsafe, nullptr, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount, nMinDepth, nMaxDepth); @@ -2854,9 +2996,9 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) throw std::runtime_error( - "fundrawtransaction \"hexstring\" ( options )\n" + "fundrawtransaction \"hexstring\" ( options iswitness )\n" "\nAdd inputs to a transaction until it has enough in value to meet its out value.\n" "This will not modify existing inputs, and will add at most one change output to the outputs.\n" "No existing outputs will be modified unless \"subtractFeeFromOutputs\" is specified.\n" @@ -2891,6 +3033,9 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) " \"CONSERVATIVE\"\n" " }\n" " for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}\n" + "3. iswitness (boolean, optional) Whether the transaction hex is a serialized witness transaction \n" + " If iswitness is not present, heuristic tests will be used in decoding\n" + "\nResult:\n" "{\n" " \"hex\": \"value\", (string) The resulting raw transaction (hex-encoded string)\n" @@ -2911,6 +3056,10 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) ObserveSafeMode(); RPCTypeCheck(request.params, {UniValue::VSTR}); + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + CCoinControl coinControl; int changePosition = -1; bool lockUnspents = false; @@ -2923,7 +3072,7 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) coinControl.fAllowWatchOnly = request.params[1].get_bool(); } else { - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VOBJ}); + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VOBJ, UniValue::VBOOL}); UniValue options = request.params[1]; @@ -2992,8 +3141,11 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) // parse hex string from parameter CMutableTransaction tx; - if (!DecodeHexTx(tx, request.params[0].get_str(), true)) + bool try_witness = request.params[2].isNull() ? true : request.params[2].get_bool(); + bool try_no_witness = request.params[2].isNull() ? true : !request.params[2].get_bool(); + if (!DecodeHexTx(tx, request.params[0].get_str(), try_no_witness, try_witness)) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); + } if (tx.vout.size() == 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "TX must have at least one output"); @@ -3121,48 +3273,57 @@ UniValue bumpfee(const JSONRPCRequest& request) } } + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); - CFeeBumper feeBump(pwallet, hash, coin_control, totalFee); - BumpFeeResult res = feeBump.getResult(); - if (res != BumpFeeResult::OK) - { + + std::vector<std::string> errors; + CAmount old_fee; + CAmount new_fee; + CMutableTransaction mtx; + feebumper::Result res = feebumper::CreateTransaction(pwallet, hash, coin_control, totalFee, errors, old_fee, new_fee, mtx); + if (res != feebumper::Result::OK) { switch(res) { - case BumpFeeResult::INVALID_ADDRESS_OR_KEY: - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, feeBump.getErrors()[0]); + case feebumper::Result::INVALID_ADDRESS_OR_KEY: + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errors[0]); break; - case BumpFeeResult::INVALID_REQUEST: - throw JSONRPCError(RPC_INVALID_REQUEST, feeBump.getErrors()[0]); + case feebumper::Result::INVALID_REQUEST: + throw JSONRPCError(RPC_INVALID_REQUEST, errors[0]); break; - case BumpFeeResult::INVALID_PARAMETER: - throw JSONRPCError(RPC_INVALID_PARAMETER, feeBump.getErrors()[0]); + case feebumper::Result::INVALID_PARAMETER: + throw JSONRPCError(RPC_INVALID_PARAMETER, errors[0]); break; - case BumpFeeResult::WALLET_ERROR: - throw JSONRPCError(RPC_WALLET_ERROR, feeBump.getErrors()[0]); + case feebumper::Result::WALLET_ERROR: + throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); break; default: - throw JSONRPCError(RPC_MISC_ERROR, feeBump.getErrors()[0]); + throw JSONRPCError(RPC_MISC_ERROR, errors[0]); break; } } // sign bumped transaction - if (!feeBump.signTransaction(pwallet)) { + if (!feebumper::SignTransaction(pwallet, mtx)) { throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); } // commit the bumped transaction - if(!feeBump.commit(pwallet)) { - throw JSONRPCError(RPC_WALLET_ERROR, feeBump.getErrors()[0]); + uint256 txid; + if (feebumper::CommitTransaction(pwallet, hash, std::move(mtx), errors, txid) != feebumper::Result::OK) { + throw JSONRPCError(RPC_WALLET_ERROR, errors[0]); } UniValue result(UniValue::VOBJ); - result.push_back(Pair("txid", feeBump.getBumpedTxId().GetHex())); - result.push_back(Pair("origfee", ValueFromAmount(feeBump.getOldFee()))); - result.push_back(Pair("fee", ValueFromAmount(feeBump.getNewFee()))); - UniValue errors(UniValue::VARR); - for (const std::string& err: feeBump.getErrors()) - errors.push_back(err); - result.push_back(Pair("errors", errors)); + result.push_back(Pair("txid", txid.GetHex())); + result.push_back(Pair("origfee", ValueFromAmount(old_fee))); + result.push_back(Pair("fee", ValueFromAmount(new_fee))); + UniValue result_errors(UniValue::VARR); + for (const std::string& error : errors) { + result_errors.push_back(error); + } + result.push_back(Pair("errors", result_errors)); return result; } @@ -3212,6 +3373,81 @@ UniValue generate(const JSONRPCRequest& request) return generateBlocks(coinbase_script, num_generate, max_tries, true); } +UniValue rescanblockchain(const JSONRPCRequest& request) +{ + CWallet * const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + if (request.fHelp || request.params.size() > 2) { + throw std::runtime_error( + "rescanblockchain (\"start_height\") (\"stop_height\")\n" + "\nRescan the local blockchain for wallet related transactions.\n" + "\nArguments:\n" + "1. \"start_height\" (numeric, optional) block height where the rescan should start\n" + "2. \"stop_height\" (numeric, optional) the last block height that should be scanned\n" + "\nResult:\n" + "{\n" + " \"start_height\" (numeric) The block height where the rescan has started. If omitted, rescan started from the genesis block.\n" + " \"stop_height\" (numeric) The height of the last rescanned block. If omitted, rescan stopped at the chain tip.\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("rescanblockchain", "100000 120000") + + HelpExampleRpc("rescanblockchain", "100000, 120000") + ); + } + + LOCK2(cs_main, pwallet->cs_wallet); + + CBlockIndex *pindexStart = chainActive.Genesis(); + CBlockIndex *pindexStop = nullptr; + if (!request.params[0].isNull()) { + pindexStart = chainActive[request.params[0].get_int()]; + if (!pindexStart) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid start_height"); + } + } + + if (!request.params[1].isNull()) { + pindexStop = chainActive[request.params[1].get_int()]; + if (!pindexStop) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stop_height"); + } + else if (pindexStop->nHeight < pindexStart->nHeight) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "stop_height must be greater then start_height"); + } + } + + // We can't rescan beyond non-pruned blocks, stop and throw an error + if (fPruneMode) { + CBlockIndex *block = pindexStop ? pindexStop : chainActive.Tip(); + while (block && block->nHeight >= pindexStart->nHeight) { + if (!(block->nStatus & BLOCK_HAVE_DATA)) { + throw JSONRPCError(RPC_MISC_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height."); + } + block = block->pprev; + } + } + + CBlockIndex *stopBlock = pwallet->ScanForWalletTransactions(pindexStart, pindexStop, true); + if (!stopBlock) { + if (pwallet->IsAbortingRescan()) { + throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted."); + } + // if we got a nullptr returned, ScanForWalletTransactions did rescan up to the requested stopindex + stopBlock = pindexStop ? pindexStop : chainActive.Tip(); + } + else { + throw JSONRPCError(RPC_MISC_ERROR, "Rescan failed. Potentially corrupted data files."); + } + + UniValue response(UniValue::VOBJ); + response.pushKV("start_height", pindexStart->nHeight); + response.pushKV("stop_height", stopBlock->nHeight); + return response; +} + extern UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue importprivkey(const JSONRPCRequest& request); @@ -3222,11 +3458,12 @@ extern UniValue importwallet(const JSONRPCRequest& request); extern UniValue importprunedfunds(const JSONRPCRequest& request); extern UniValue removeprunedfunds(const JSONRPCRequest& request); extern UniValue importmulti(const JSONRPCRequest& request); +extern UniValue rescanblockchain(const JSONRPCRequest& request); static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- - { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options"} }, + { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options","iswitness"} }, { "hidden", "resendwallettransactions", &resendwallettransactions, {} }, { "wallet", "abandontransaction", &abandontransaction, {"txid"} }, { "wallet", "abortrescan", &abortrescan, {} }, @@ -3241,8 +3478,8 @@ static const CRPCCommand commands[] = { "wallet", "getaccount", &getaccount, {"address"} }, { "wallet", "getaddressesbyaccount", &getaddressesbyaccount, {"account"} }, { "wallet", "getbalance", &getbalance, {"account","minconf","include_watchonly"} }, - { "wallet", "getnewaddress", &getnewaddress, {"account"} }, - { "wallet", "getrawchangeaddress", &getrawchangeaddress, {} }, + { "wallet", "getnewaddress", &getnewaddress, {"account","address_type"} }, + { "wallet", "getrawchangeaddress", &getrawchangeaddress, {"address_type"} }, { "wallet", "getreceivedbyaccount", &getreceivedbyaccount, {"account","minconf"} }, { "wallet", "getreceivedbyaddress", &getreceivedbyaddress, {"address","minconf"} }, { "wallet", "gettransaction", &gettransaction, {"txid","include_watchonly"} }, @@ -3276,6 +3513,7 @@ static const CRPCCommand commands[] = { "wallet", "walletpassphrasechange", &walletpassphrasechange, {"oldpassphrase","newpassphrase"} }, { "wallet", "walletpassphrase", &walletpassphrase, {"passphrase","timeout"} }, { "wallet", "removeprunedfunds", &removeprunedfunds, {"txid"} }, + { "wallet", "rescanblockchain", &rescanblockchain, {"start_height", "stop_height"} }, { "generating", "generate", &generate, {"nblocks","maxtries"} }, }; diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 14e51610d9..77f7b42b23 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/accounting_tests.cpp b/src/wallet/test/accounting_tests.cpp index 330878ceb5..cafd69d075 100644 --- a/src/wallet/test/accounting_tests.cpp +++ b/src/wallet/test/accounting_tests.cpp @@ -1,27 +1,25 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/wallet.h" +#include <wallet/wallet.h> -#include "wallet/test/wallet_test_fixture.h" +#include <wallet/test/wallet_test_fixture.h> #include <stdint.h> #include <boost/test/unit_test.hpp> -extern CWallet* pwalletMain; - BOOST_FIXTURE_TEST_SUITE(accounting_tests, WalletTestingSetup) static void -GetResults(std::map<CAmount, CAccountingEntry>& results) +GetResults(CWallet *wallet, std::map<CAmount, CAccountingEntry>& results) { std::list<CAccountingEntry> aes; results.clear(); - BOOST_CHECK(pwalletMain->ReorderTransactions() == DB_LOAD_OK); - pwalletMain->ListAccountCreditDebit("", aes); + BOOST_CHECK(wallet->ReorderTransactions() == DB_LOAD_OK); + wallet->ListAccountCreditDebit("", aes); for (CAccountingEntry& ae : aes) { results[ae.nOrderPos] = ae; @@ -54,7 +52,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) ae.strOtherAccount = "c"; pwalletMain->AddAccountingEntry(ae); - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(pwalletMain->nOrderPosNext == 3); BOOST_CHECK(2 == results.size()); @@ -70,7 +68,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) ae.nOrderPos = pwalletMain->IncOrderPosNext(); pwalletMain->AddAccountingEntry(ae); - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(results.size() == 3); BOOST_CHECK(pwalletMain->nOrderPosNext == 4); @@ -83,7 +81,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) wtx.mapValue["comment"] = "y"; { - CMutableTransaction tx(wtx); + CMutableTransaction tx(*wtx.tx); --tx.nLockTime; // Just to change the hash :) wtx.SetTx(MakeTransactionRef(std::move(tx))); } @@ -93,7 +91,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) wtx.mapValue["comment"] = "x"; { - CMutableTransaction tx(wtx); + CMutableTransaction tx(*wtx.tx); --tx.nLockTime; // Just to change the hash :) wtx.SetTx(MakeTransactionRef(std::move(tx))); } @@ -102,7 +100,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) vpwtx[2]->nTimeReceived = (unsigned int)1333333329; vpwtx[2]->nOrderPos = -1; - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(results.size() == 3); BOOST_CHECK(pwalletMain->nOrderPosNext == 6); @@ -120,7 +118,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) ae.nOrderPos = -1; pwalletMain->AddAccountingEntry(ae); - GetResults(results); + GetResults(pwalletMain.get(), results); BOOST_CHECK(results.size() == 4); BOOST_CHECK(pwalletMain->nOrderPosNext == 7); diff --git a/src/wallet/test/crypto_tests.cpp b/src/wallet/test/crypto_tests.cpp index f4dabc50c0..89b2c4e796 100644 --- a/src/wallet/test/crypto_tests.cpp +++ b/src/wallet/test/crypto_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2014-2016 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "test/test_bitcoin.h" -#include "utilstrencodings.h" -#include "wallet/crypter.h" +#include <test/test_bitcoin.h> +#include <utilstrencodings.h> +#include <wallet/crypter.h> #include <vector> diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index e2f48c45ab..7797f85f07 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -1,14 +1,11 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/test/wallet_test_fixture.h" +#include <wallet/test/wallet_test_fixture.h> -#include "rpc/server.h" -#include "wallet/db.h" -#include "wallet/wallet.h" - -CWallet *pwalletMain; +#include <rpc/server.h> +#include <wallet/db.h> WalletTestingSetup::WalletTestingSetup(const std::string& chainName): TestingSetup(chainName) @@ -16,19 +13,19 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName): bitdb.MakeMock(); bool fFirstRun; + g_address_type = OUTPUT_TYPE_DEFAULT; + g_change_type = OUTPUT_TYPE_DEFAULT; std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); - pwalletMain = new CWallet(std::move(dbw)); + pwalletMain = MakeUnique<CWallet>(std::move(dbw)); pwalletMain->LoadWallet(fFirstRun); - RegisterValidationInterface(pwalletMain); + RegisterValidationInterface(pwalletMain.get()); RegisterWalletRPCCommands(tableRPC); } WalletTestingSetup::~WalletTestingSetup() { - UnregisterValidationInterface(pwalletMain); - delete pwalletMain; - pwalletMain = nullptr; + UnregisterValidationInterface(pwalletMain.get()); bitdb.Flush(true); bitdb.Reset(); diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index 9373b7907c..c03aec7f87 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -1,17 +1,21 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_WALLET_TEST_FIXTURE_H #define BITCOIN_WALLET_TEST_FIXTURE_H -#include "test/test_bitcoin.h" +#include <test/test_bitcoin.h> + +#include <wallet/wallet.h> /** Testing setup and teardown for wallet. */ struct WalletTestingSetup: public TestingSetup { explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); ~WalletTestingSetup(); + + std::unique_ptr<CWallet> pwalletMain; }; #endif diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 5ebacd57d3..7b3c283f37 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -1,26 +1,24 @@ -// Copyright (c) 2012-2016 The Bitcoin Core developers +// Copyright (c) 2012-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/wallet.h" +#include <wallet/wallet.h> #include <set> #include <stdint.h> #include <utility> #include <vector> -#include "consensus/validation.h" -#include "rpc/server.h" -#include "test/test_bitcoin.h" -#include "validation.h" -#include "wallet/coincontrol.h" -#include "wallet/test/wallet_test_fixture.h" +#include <consensus/validation.h> +#include <rpc/server.h> +#include <test/test_bitcoin.h> +#include <validation.h> +#include <wallet/coincontrol.h> +#include <wallet/test/wallet_test_fixture.h> #include <boost/test/unit_test.hpp> #include <univalue.h> -extern CWallet* pwalletMain; - extern UniValue importmulti(const JSONRPCRequest& request); extern UniValue dumpwallet(const JSONRPCRequest& request); extern UniValue importwallet(const JSONRPCRequest& request); @@ -372,8 +370,6 @@ static void AddKey(CWallet& wallet, const CKey& key) BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) { - LOCK(cs_main); - // Cap last block file size, and mine new block in a new block file. CBlockIndex* const nullBlock = nullptr; CBlockIndex* oldTip = chainActive.Tip(); @@ -381,12 +377,14 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = chainActive.Tip(); + LOCK(cs_main); + // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { CWallet wallet; AddKey(wallet, coinbaseKey); - BOOST_CHECK_EQUAL(nullBlock, wallet.ScanForWalletTransactions(oldTip)); + BOOST_CHECK_EQUAL(nullBlock, wallet.ScanForWalletTransactions(oldTip, nullptr)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 100 * COIN); } @@ -399,7 +397,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) { CWallet wallet; AddKey(wallet, coinbaseKey); - BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip)); + BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip, nullptr)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 50 * COIN); } @@ -449,8 +447,6 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) // than or equal to key birthday. BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) { - LOCK(cs_main); - // Create two blocks with same timestamp to verify that importwallet rescan // will pick up both blocks, not just the first. const int64_t BLOCK_TIME = chainActive.Tip()->GetBlockTimeMax() + 5; @@ -464,6 +460,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) SetMockTime(KEY_TIME); coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + LOCK(cs_main); + // Import key into wallet and call dumpwallet to create backup file. { CWallet wallet; @@ -489,6 +487,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) vpwallets[0] = &wallet; ::importwallet(request); + LOCK(wallet.cs_wallet); BOOST_CHECK_EQUAL(wallet.mapWallet.size(), 3); BOOST_CHECK_EQUAL(coinbaseTxns.size(), 103); for (size_t i = 0; i < coinbaseTxns.size(); ++i) { @@ -534,6 +533,7 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 SetMockTime(mockTime); CBlockIndex* block = nullptr; if (blockTime > 0) { + LOCK(cs_main); auto inserted = mapBlockIndex.emplace(GetRandHash(), new CBlockIndex); assert(inserted.second); const uint256& hash = inserted.first->first; @@ -547,6 +547,7 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 wtx.SetMerkleBranch(block, 0); } wallet.AddToWallet(wtx); + LOCK(wallet.cs_wallet); return wallet.mapWallet.at(wtx.GetHash()).nTimeSmart; } @@ -583,6 +584,7 @@ BOOST_AUTO_TEST_CASE(ComputeTimeSmart) BOOST_AUTO_TEST_CASE(LoadReceiveRequests) { CTxDestination dest = CKeyID(); + LOCK(pwalletMain->cs_wallet); pwalletMain->AddDestData(dest, "misc", "val_misc"); pwalletMain->AddDestData(dest, "rr0", "val_rr0"); pwalletMain->AddDestData(dest, "rr1", "val_rr1"); @@ -600,11 +602,13 @@ public: { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); ::bitdb.MakeMock(); + g_address_type = OUTPUT_TYPE_DEFAULT; + g_change_type = OUTPUT_TYPE_DEFAULT; wallet.reset(new CWallet(std::unique_ptr<CWalletDBWrapper>(new CWalletDBWrapper(&bitdb, "wallet_test.dat")))); bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); - wallet->ScanForWalletTransactions(chainActive.Genesis()); + wallet->ScanForWalletTransactions(chainActive.Genesis(), nullptr); } ~ListCoinsTestingSetup() @@ -625,9 +629,15 @@ public: BOOST_CHECK(wallet->CreateTransaction({recipient}, wtx, reservekey, fee, changePos, error, dummy)); CValidationState state; BOOST_CHECK(wallet->CommitTransaction(wtx, reservekey, nullptr, state)); + CMutableTransaction blocktx; + { + LOCK(wallet->cs_wallet); + blocktx = CMutableTransaction(*wallet->mapWallet.at(wtx.GetHash()).tx); + } + CreateAndProcessBlock({CMutableTransaction(blocktx)}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + LOCK(wallet->cs_wallet); auto it = wallet->mapWallet.find(wtx.GetHash()); BOOST_CHECK(it != wallet->mapWallet.end()); - CreateAndProcessBlock({CMutableTransaction(*it->second.tx)}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); it->second.SetMerkleBranch(chainActive.Tip(), 1); return it->second; } @@ -638,7 +648,6 @@ public: BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) { std::string coinbaseAddress = coinbaseKey.GetPubKey().GetID().ToString(); - LOCK2(cs_main, wallet->cs_wallet); // Confirm ListCoins initially returns 1 coin grouped under coinbaseKey // address. @@ -666,6 +675,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) BOOST_CHECK_EQUAL(available.size(), 2); for (const auto& group : list) { for (const auto& coin : group.second) { + LOCK(wallet->cs_wallet); wallet->LockCoin(COutPoint(coin.tx->GetHash(), coin.i)); } } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 925b474d73..da721ea008 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1,38 +1,37 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/wallet.h" - -#include "base58.h" -#include "checkpoints.h" -#include "chain.h" -#include "wallet/coincontrol.h" -#include "consensus/consensus.h" -#include "consensus/validation.h" -#include "fs.h" -#include "init.h" -#include "key.h" -#include "keystore.h" -#include "validation.h" -#include "net.h" -#include "policy/fees.h" -#include "policy/policy.h" -#include "policy/rbf.h" -#include "primitives/block.h" -#include "primitives/transaction.h" -#include "script/script.h" -#include "script/sign.h" -#include "scheduler.h" -#include "timedata.h" -#include "txmempool.h" -#include "util.h" -#include "ui_interface.h" -#include "utilmoneystr.h" -#include "wallet/fees.h" +#include <wallet/wallet.h> + +#include <base58.h> +#include <checkpoints.h> +#include <chain.h> +#include <wallet/coincontrol.h> +#include <consensus/consensus.h> +#include <consensus/validation.h> +#include <fs.h> +#include <wallet/init.h> +#include <key.h> +#include <keystore.h> +#include <validation.h> +#include <net.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <scheduler.h> +#include <timedata.h> +#include <txmempool.h> +#include <util.h> +#include <utilmoneystr.h> +#include <wallet/fees.h> #include <assert.h> +#include <future> #include <boost/algorithm/string/replace.hpp> #include <boost/thread.hpp> @@ -43,6 +42,8 @@ CFeeRate payTxFee(DEFAULT_TRANSACTION_FEE); unsigned int nTxConfirmTarget = DEFAULT_TX_CONFIRM_TARGET; bool bSpendZeroConfChange = DEFAULT_SPEND_ZEROCONF_CHANGE; bool fWalletRbf = DEFAULT_WALLET_RBF; +OutputType g_address_type = OUTPUT_TYPE_NONE; +OutputType g_change_type = OUTPUT_TYPE_NONE; const char * DEFAULT_WALLET_DAT = "wallet.dat"; const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; @@ -283,7 +284,7 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, } } -bool CWallet::LoadKeyMetadata(const CTxDestination& keyID, const CKeyMetadata &meta) +bool CWallet::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &meta) { AssertLockHeld(cs_wallet); // mapKeyMetadata UpdateTimeFirstKey(meta.nCreateTime); @@ -291,6 +292,14 @@ bool CWallet::LoadKeyMetadata(const CTxDestination& keyID, const CKeyMetadata &m return true; } +bool CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &meta) +{ + AssertLockHeld(cs_wallet); // m_script_metadata + UpdateTimeFirstKey(meta.nCreateTime); + m_script_metadata[script_id] = meta; + return true; +} + bool CWallet::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) { return CCryptoKeyStore::AddCryptedKey(vchPubKey, vchCryptedSecret); @@ -339,7 +348,7 @@ bool CWallet::AddWatchOnly(const CScript& dest) { if (!CCryptoKeyStore::AddWatchOnly(dest)) return false; - const CKeyMetadata& meta = mapKeyMetadata[CScriptID(dest)]; + const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)]; UpdateTimeFirstKey(meta.nCreateTime); NotifyWatchonlyChanged(true); return CWalletDB(*dbw).WriteWatchOnly(dest, meta); @@ -347,7 +356,7 @@ bool CWallet::AddWatchOnly(const CScript& dest) bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) { - mapKeyMetadata[CScriptID(dest)].nCreateTime = nCreateTime; + m_script_metadata[CScriptID(dest)].nCreateTime = nCreateTime; return AddWatchOnly(dest); } @@ -532,6 +541,9 @@ void CWallet::SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator> ran copyFrom = &mapWallet[hash]; } } + + assert(copyFrom); + // Now copy data from copyFrom to rest: for (TxSpends::iterator it = range.first; it != range.second; ++it) { @@ -701,9 +713,9 @@ DBErrors CWallet::ReorderTransactions() typedef std::multimap<int64_t, TxPair > TxItems; TxItems txByTime; - for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (auto& entry : mapWallet) { - CWalletTx* wtx = &((*it).second); + CWalletTx* wtx = &entry.second; txByTime.insert(std::make_pair(wtx->nTimeReceived, TxPair(wtx, nullptr))); } std::list<CAccountingEntry> acentries; @@ -811,7 +823,7 @@ bool CWallet::AccountMove(std::string strFrom, std::string strTo, CAmount nAmoun return true; } -bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bForceNew) +bool CWallet::GetAccountDestination(CTxDestination &dest, std::string strAccount, bool bForceNew) { CWalletDB walletdb(*dbw); @@ -822,8 +834,8 @@ bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bFo if (!account.vchPubKey.IsValid()) bForceNew = true; else { - // Check if the current key has been used - CScript scriptPubKey = GetScriptForDestination(account.vchPubKey.GetID()); + // Check if the current key has been used (TODO: check other addresses with the same key) + CScript scriptPubKey = GetScriptForDestination(GetDestinationForKey(account.vchPubKey, g_address_type)); for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end() && account.vchPubKey.IsValid(); ++it) @@ -840,12 +852,14 @@ bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bFo if (!GetKeyFromPool(account.vchPubKey, false)) return false; - SetAddressBook(account.vchPubKey.GetID(), strAccount, "receive"); + LearnRelatedScripts(account.vchPubKey, g_address_type); + dest = GetDestinationForKey(account.vchPubKey, g_address_type); + SetAddressBook(dest, strAccount, "receive"); walletdb.WriteAccount(strAccount, account); + } else { + dest = GetDestinationForKey(account.vchPubKey, g_address_type); } - pubKey = account.vchPubKey; - return true; } @@ -1214,6 +1228,19 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, const CBlockIndex *pin void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { LOCK2(cs_main, cs_wallet); SyncTransaction(ptx); + + auto it = mapWallet.find(ptx->GetHash()); + if (it != mapWallet.end()) { + it->second.fInMempool = true; + } +} + +void CWallet::TransactionRemovedFromMempool(const CTransactionRef &ptx) { + LOCK(cs_wallet); + auto it = mapWallet.find(ptx->GetHash()); + if (it != mapWallet.end()) { + it->second.fInMempool = false; + } } void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { @@ -1228,10 +1255,14 @@ void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const for (const CTransactionRef& ptx : vtxConflicted) { SyncTransaction(ptx); + TransactionRemovedFromMempool(ptx); } for (size_t i = 0; i < pblock->vtx.size(); i++) { SyncTransaction(pblock->vtx[i], pindex, i); + TransactionRemovedFromMempool(pblock->vtx[i]); } + + m_last_block_processed = pindex; } void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { @@ -1244,6 +1275,31 @@ void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { +void CWallet::BlockUntilSyncedToCurrentChain() { + AssertLockNotHeld(cs_main); + AssertLockNotHeld(cs_wallet); + + { + // Skip the queue-draining stuff if we know we're caught up with + // chainActive.Tip()... + // We could also take cs_wallet here, and call m_last_block_processed + // protected by cs_wallet instead of cs_main, but as long as we need + // cs_main here anyway, its easier to just call it cs_main-protected. + LOCK(cs_main); + const CBlockIndex* initialChainTip = chainActive.Tip(); + + if (m_last_block_processed->GetAncestor(initialChainTip->nHeight) == initialChainTip) { + return; + } + } + + // ...otherwise put a callback in the validation interface queue and wait + // for the queue to drain enough to execute it (indicating we are caught up + // at least with the time we entered this function). + SyncWithValidationInterfaceQueue(); +} + + isminetype CWallet::IsMine(const CTxIn &txin) const { { @@ -1568,7 +1624,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, bool update) LogPrintf("%s: Rescanning last %i blocks\n", __func__, startBlock ? chainActive.Height() - startBlock->nHeight + 1 : 0); if (startBlock) { - const CBlockIndex* const failedBlock = ScanForWalletTransactions(startBlock, update); + const CBlockIndex* const failedBlock = ScanForWalletTransactions(startBlock, nullptr, update); if (failedBlock) { return failedBlock->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1; } @@ -1584,12 +1640,19 @@ int64_t CWallet::RescanFromTime(int64_t startTime, bool update) * Returns null if scan was successful. Otherwise, if a complete rescan was not * possible (due to pruning or corruption), returns pointer to the most recent * block that could not be scanned. + * + * If pindexStop is not a nullptr, the scan will stop at the block-index + * defined by pindexStop */ -CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) +CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop, bool fUpdate) { int64_t nNow = GetTime(); const CChainParams& chainParams = Params(); + if (pindexStop) { + assert(pindexStop->nHeight >= pindexStart->nHeight); + } + CBlockIndex* pindex = pindexStart; CBlockIndex* ret = nullptr; { @@ -1617,6 +1680,9 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f } else { ret = pindex; } + if (pindex == pindexStop) { + break; + } pindex = chainActive.Next(pindex); } if (pindex && fAbortRescan) { @@ -1652,11 +1718,8 @@ void CWallet::ReacceptWalletTransactions() } // Try to add wallet transactions to memory pool - for (std::pair<const int64_t, CWalletTx*>& item : mapSorted) - { + for (std::pair<const int64_t, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *(item.second); - - LOCK(mempool.cs); CValidationState state; wtx.AcceptToMemoryPool(maxTxFee, state); } @@ -1708,7 +1771,7 @@ CAmount CWalletTx::GetDebit(const isminefilter& filter) const debit += nDebitCached; else { - nDebitCached = pwallet->GetDebit(*this, ISMINE_SPENDABLE); + nDebitCached = pwallet->GetDebit(*tx, ISMINE_SPENDABLE); fDebitCached = true; debit += nDebitCached; } @@ -1719,7 +1782,7 @@ CAmount CWalletTx::GetDebit(const isminefilter& filter) const debit += nWatchDebitCached; else { - nWatchDebitCached = pwallet->GetDebit(*this, ISMINE_WATCH_ONLY); + nWatchDebitCached = pwallet->GetDebit(*tx, ISMINE_WATCH_ONLY); fWatchDebitCached = true; debit += nWatchDebitCached; } @@ -1741,7 +1804,7 @@ CAmount CWalletTx::GetCredit(const isminefilter& filter) const credit += nCreditCached; else { - nCreditCached = pwallet->GetCredit(*this, ISMINE_SPENDABLE); + nCreditCached = pwallet->GetCredit(*tx, ISMINE_SPENDABLE); fCreditCached = true; credit += nCreditCached; } @@ -1752,7 +1815,7 @@ CAmount CWalletTx::GetCredit(const isminefilter& filter) const credit += nWatchCreditCached; else { - nWatchCreditCached = pwallet->GetCredit(*this, ISMINE_WATCH_ONLY); + nWatchCreditCached = pwallet->GetCredit(*tx, ISMINE_WATCH_ONLY); fWatchCreditCached = true; credit += nWatchCreditCached; } @@ -1766,7 +1829,7 @@ CAmount CWalletTx::GetImmatureCredit(bool fUseCache) const { if (fUseCache && fImmatureCreditCached) return nImmatureCreditCached; - nImmatureCreditCached = pwallet->GetCredit(*this, ISMINE_SPENDABLE); + nImmatureCreditCached = pwallet->GetCredit(*tx, ISMINE_SPENDABLE); fImmatureCreditCached = true; return nImmatureCreditCached; } @@ -1804,13 +1867,13 @@ CAmount CWalletTx::GetAvailableCredit(bool fUseCache) const return nCredit; } -CAmount CWalletTx::GetImmatureWatchOnlyCredit(const bool& fUseCache) const +CAmount CWalletTx::GetImmatureWatchOnlyCredit(const bool fUseCache) const { if (IsCoinBase() && GetBlocksToMaturity() > 0 && IsInMainChain()) { if (fUseCache && fImmatureWatchCreditCached) return nImmatureWatchCreditCached; - nImmatureWatchCreditCached = pwallet->GetCredit(*this, ISMINE_WATCH_ONLY); + nImmatureWatchCreditCached = pwallet->GetCredit(*tx, ISMINE_WATCH_ONLY); fImmatureWatchCreditCached = true; return nImmatureWatchCreditCached; } @@ -1818,7 +1881,7 @@ CAmount CWalletTx::GetImmatureWatchOnlyCredit(const bool& fUseCache) const return 0; } -CAmount CWalletTx::GetAvailableWatchOnlyCredit(const bool& fUseCache) const +CAmount CWalletTx::GetAvailableWatchOnlyCredit(const bool fUseCache) const { if (pwallet == nullptr) return 0; @@ -1851,21 +1914,20 @@ CAmount CWalletTx::GetChange() const { if (fChangeCached) return nChangeCached; - nChangeCached = pwallet->GetChange(*this); + nChangeCached = pwallet->GetChange(*tx); fChangeCached = true; return nChangeCached; } bool CWalletTx::InMempool() const { - LOCK(mempool.cs); - return mempool.exists(GetHash()); + return fInMempool; } bool CWalletTx::IsTrusted() const { // Quick answer in most cases - if (!CheckFinalTx(*this)) + if (!CheckFinalTx(*tx)) return false; int nDepth = GetDepthInMainChain(); if (nDepth >= 1) @@ -1966,9 +2028,9 @@ CAmount CWallet::GetBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; if (pcoin->IsTrusted()) nTotal += pcoin->GetAvailableCredit(); } @@ -1982,9 +2044,9 @@ CAmount CWallet::GetUnconfirmedBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; if (!pcoin->IsTrusted() && pcoin->GetDepthInMainChain() == 0 && pcoin->InMempool()) nTotal += pcoin->GetAvailableCredit(); } @@ -1997,9 +2059,9 @@ CAmount CWallet::GetImmatureBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; nTotal += pcoin->GetImmatureCredit(); } } @@ -2011,9 +2073,9 @@ CAmount CWallet::GetWatchOnlyBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; if (pcoin->IsTrusted()) nTotal += pcoin->GetAvailableWatchOnlyCredit(); } @@ -2027,9 +2089,9 @@ CAmount CWallet::GetUnconfirmedWatchOnlyBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; if (!pcoin->IsTrusted() && pcoin->GetDepthInMainChain() == 0 && pcoin->InMempool()) nTotal += pcoin->GetAvailableWatchOnlyCredit(); } @@ -2042,9 +2104,9 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const CAmount nTotal = 0; { LOCK2(cs_main, cs_wallet); - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const CWalletTx* pcoin = &(*it).second; + const CWalletTx* pcoin = &entry.second; nTotal += pcoin->GetImmatureWatchOnlyCredit(); } } @@ -2109,7 +2171,7 @@ CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const return balance; } -void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t &nMaximumCount, const int &nMinDepth, const int &nMaxDepth) const +void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t nMaximumCount, const int nMinDepth, const int nMaxDepth) const { vCoins.clear(); @@ -2118,12 +2180,12 @@ void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CAmount nTotal = 0; - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) + for (const auto& entry : mapWallet) { - const uint256& wtxid = it->first; - const CWalletTx* pcoin = &(*it).second; + const uint256& wtxid = entry.first; + const CWalletTx* pcoin = &entry.second; - if (!CheckFinalTx(*pcoin)) + if (!CheckFinalTx(*pcoin->tx)) continue; if (pcoin->IsCoinBase() && pcoin->GetBlocksToMaturity() > 0) @@ -2182,10 +2244,10 @@ void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const if (pcoin->tx->vout[i].nValue < nMinimumAmount || pcoin->tx->vout[i].nValue > nMaximumAmount) continue; - if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint((*it).first, i))) + if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint(entry.first, i))) continue; - if (IsLockedCoin((*it).first, i)) + if (IsLockedCoin(entry.first, i)) continue; if (IsSpent(wtxid, i)) @@ -2532,9 +2594,8 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC { std::vector<CRecipient> vecSend; - // Turn the txout set into a CRecipient vector - for (size_t idx = 0; idx < tx.vout.size(); idx++) - { + // Turn the txout set into a CRecipient vector. + for (size_t idx = 0; idx < tx.vout.size(); idx++) { const CTxOut& txOut = tx.vout[idx]; CRecipient recipient = {txOut.scriptPubKey, txOut.nValue, setSubtractFeeFromOutputs.count(idx) == 1}; vecSend.push_back(recipient); @@ -2542,8 +2603,13 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC coinControl.fAllowOtherInputs = true; - for (const CTxIn& txin : tx.vin) + for (const CTxIn& txin : tx.vin) { coinControl.Select(txin.prevout); + } + + // Acquire the locks to prevent races to the new locked unspents between the + // CreateTransaction call and LockCoin calls (when lockUnspents is true). + LOCK2(cs_main, cs_wallet); CReserveKey reservekey(this); CWalletTx wtx; @@ -2553,31 +2619,28 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC if (nChangePosInOut != -1) { tx.vout.insert(tx.vout.begin() + nChangePosInOut, wtx.tx->vout[nChangePosInOut]); - // we don't have the normal Create/Commit cycle, and don't want to risk reusing change, - // so just remove the key from the keypool here. + // We don't have the normal Create/Commit cycle, and don't want to risk + // reusing change, so just remove the key from the keypool here. reservekey.KeepKey(); } - // Copy output sizes from new transaction; they may have had the fee subtracted from them - for (unsigned int idx = 0; idx < tx.vout.size(); idx++) + // Copy output sizes from new transaction; they may have had the fee + // subtracted from them. + for (unsigned int idx = 0; idx < tx.vout.size(); idx++) { tx.vout[idx].nValue = wtx.tx->vout[idx].nValue; + } - // Add new txins (keeping original txin scriptSig/order) - for (const CTxIn& txin : wtx.tx->vin) - { - if (!coinControl.IsSelected(txin.prevout)) - { + // Add new txins while keeping original txin scriptSig/order. + for (const CTxIn& txin : wtx.tx->vin) { + if (!coinControl.IsSelected(txin.prevout)) { tx.vin.push_back(txin); - if (lockUnspents) - { - LOCK2(cs_main, cs_wallet); - LockCoin(txin.prevout); + if (lockUnspents) { + LockCoin(txin.prevout); } } } - return true; } @@ -2676,7 +2739,8 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT return false; } - scriptChange = GetScriptForDestination(vchPubKey.GetID()); + LearnRelatedScripts(vchPubKey, g_change_type); + scriptChange = GetScriptForDestination(GetDestinationForKey(vchPubKey, g_change_type)); } CTxOut change_prototype_txout(0, scriptChange); size_t change_prototype_size = GetSerializeSize(change_prototype_txout, SER_DISK, 0); @@ -2704,6 +2768,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT if (recipient.fSubtractFeeFromAmount) { + assert(nSubtractFeeFromAmount != 0); txout.nValue -= nFeeRet / nSubtractFeeFromAmount; // Subtract fee equally from each selected recipient if (fFirst) // first receiver pays the remainder not divisible by output count @@ -2904,7 +2969,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT wtxNew.SetTx(MakeTransactionRef(std::move(txNew))); // Limit size - if (GetTransactionWeight(wtxNew) >= MAX_STANDARD_TX_WEIGHT) + if (GetTransactionWeight(*wtxNew.tx) >= MAX_STANDARD_TX_WEIGHT) { strFailReason = _("Transaction too large"); return false; @@ -2966,14 +3031,18 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon // Track how many getdata requests our transaction gets mapRequestCount[wtxNew.GetHash()] = 0; + // Get the inserted-CWalletTx from mapWallet so that the + // fInMempool flag is cached properly + CWalletTx& wtx = mapWallet[wtxNew.GetHash()]; + if (fBroadcastTransactions) { // Broadcast - if (!wtxNew.AcceptToMemoryPool(maxTxFee, state)) { + if (!wtx.AcceptToMemoryPool(maxTxFee, state)) { LogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", state.GetRejectReason()); // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. } else { - wtxNew.RelayWalletTransaction(connman); + wtx.RelayWalletTransaction(connman); } } } @@ -3560,6 +3629,7 @@ void CWallet::MarkReserveKeysAsUsed(int64_t keypool_id) if (walletdb.ReadPool(index, keypool)) { //TODO: This should be unnecessary m_pool_key_to_index.erase(keypool.vchPubKey.GetID()); } + LearnAllRelatedScripts(keypool.vchPubKey); walletdb.ErasePool(index); LogPrintf("keypool index %d removed\n", index); it = setKeyPool->erase(it); @@ -3640,9 +3710,9 @@ void CWallet::GetKeyBirthTimes(std::map<CTxDestination, int64_t> &mapKeyBirth) c // find first block that affects those keys, if there are any left std::vector<CKeyID> vAffected; - for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); it++) { + for (const auto& entry : mapWallet) { // iterate over all wallet transactions... - const CWalletTx &wtx = (*it).second; + const CWalletTx &wtx = entry.second; BlockMap::const_iterator blit = mapBlockIndex.find(wtx.hashBlock); if (blit != mapBlockIndex.end() && chainActive.Contains(blit->second)) { // ... which are already in a block @@ -3662,8 +3732,8 @@ void CWallet::GetKeyBirthTimes(std::map<CTxDestination, int64_t> &mapKeyBirth) c } // Extract block timestamps for those keys - for (std::map<CKeyID, CBlockIndex*>::const_iterator it = mapKeyFirstBlock.begin(); it != mapKeyFirstBlock.end(); it++) - mapKeyBirth[it->first] = it->second->GetBlockTime() - TIMESTAMP_WINDOW; // block times can be 2h off + for (const auto& entry : mapKeyFirstBlock) + mapKeyBirth[entry.first] = entry.second->GetBlockTime() - TIMESTAMP_WINDOW; // block times can be 2h off } /** @@ -3792,7 +3862,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) uiInterface.InitMessage(_("Zapping all transactions from wallet...")); std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile)); - std::unique_ptr<CWallet> tempWallet(new CWallet(std::move(dbw))); + std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(std::move(dbw)); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DB_LOAD_OK) { InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); @@ -3870,7 +3940,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) // Top up the keypool if (!walletInstance->TopUpKeyPool()) { InitError(_("Unable to generate initial keys") += "\n"); - return NULL; + return nullptr; } walletInstance->SetBestChain(chainActive.GetLocator()); @@ -3889,8 +3959,6 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) LogPrintf(" wallet %15dms\n", GetTimeMillis() - nStart); - RegisterValidationInterface(walletInstance); - // Try to top up keypool. No-op if the wallet is locked. walletInstance->TopUpKeyPool(); @@ -3902,6 +3970,10 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) if (walletdb.ReadBestBlock(locator)) pindexRescan = FindForkInGlobalIndex(chainActive, locator); } + + walletInstance->m_last_block_processed = chainActive.Tip(); + RegisterValidationInterface(walletInstance); + if (chainActive.Tip() && chainActive.Tip() != pindexRescan) { //We can't rescan beyond non-pruned blocks, stop and throw an error @@ -3929,7 +4001,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) } nStart = GetTimeMillis(); - walletInstance->ScanForWalletTransactions(pindexRescan, true); + walletInstance->ScanForWalletTransactions(pindexRescan, nullptr, true); LogPrintf(" rescan %15dms\n", GetTimeMillis() - nStart); walletInstance->SetBestChain(chainActive.GetLocator()); walletInstance->dbw->IncrementUpdateCounter(); @@ -4045,8 +4117,123 @@ int CMerkleTx::GetBlocksToMaturity() const } -bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) +bool CWalletTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) { - return ::AcceptToMemoryPool(mempool, state, tx, nullptr /* pfMissingInputs */, + // Quick check to avoid re-setting fInMempool to false + if (mempool.exists(tx->GetHash())) { + return false; + } + + // We must set fInMempool here - while it will be re-set to true by the + // entered-mempool callback, if we did not there would be a race where a + // user could call sendmoney in a loop and hit spurious out of funds errors + // because we think that the transaction they just generated's change is + // unavailable as we're not yet aware its in mempool. + bool ret = ::AcceptToMemoryPool(mempool, state, tx, nullptr /* pfMissingInputs */, nullptr /* plTxnReplaced */, false /* bypass_limits */, nAbsurdFee); + fInMempool = ret; + return ret; +} + +static const std::string OUTPUT_TYPE_STRING_LEGACY = "legacy"; +static const std::string OUTPUT_TYPE_STRING_P2SH_SEGWIT = "p2sh-segwit"; +static const std::string OUTPUT_TYPE_STRING_BECH32 = "bech32"; + +OutputType ParseOutputType(const std::string& type, OutputType default_type) +{ + if (type.empty()) { + return default_type; + } else if (type == OUTPUT_TYPE_STRING_LEGACY) { + return OUTPUT_TYPE_LEGACY; + } else if (type == OUTPUT_TYPE_STRING_P2SH_SEGWIT) { + return OUTPUT_TYPE_P2SH_SEGWIT; + } else if (type == OUTPUT_TYPE_STRING_BECH32) { + return OUTPUT_TYPE_BECH32; + } else { + return OUTPUT_TYPE_NONE; + } +} + +const std::string& FormatOutputType(OutputType type) +{ + switch (type) { + case OUTPUT_TYPE_LEGACY: return OUTPUT_TYPE_STRING_LEGACY; + case OUTPUT_TYPE_P2SH_SEGWIT: return OUTPUT_TYPE_STRING_P2SH_SEGWIT; + case OUTPUT_TYPE_BECH32: return OUTPUT_TYPE_STRING_BECH32; + default: assert(false); + } +} + +void CWallet::LearnRelatedScripts(const CPubKey& key, OutputType type) +{ + if (key.IsCompressed() && (type == OUTPUT_TYPE_P2SH_SEGWIT || type == OUTPUT_TYPE_BECH32)) { + CTxDestination witdest = WitnessV0KeyHash(key.GetID()); + CScript witprog = GetScriptForDestination(witdest); + // Make sure the resulting program is solvable. + assert(IsSolvable(*this, witprog)); + AddCScript(witprog); + } +} + +void CWallet::LearnAllRelatedScripts(const CPubKey& key) +{ + // OUTPUT_TYPE_P2SH_SEGWIT always adds all necessary scripts for all types. + LearnRelatedScripts(key, OUTPUT_TYPE_P2SH_SEGWIT); +} + +CTxDestination GetDestinationForKey(const CPubKey& key, OutputType type) +{ + switch (type) { + case OUTPUT_TYPE_LEGACY: return key.GetID(); + case OUTPUT_TYPE_P2SH_SEGWIT: + case OUTPUT_TYPE_BECH32: { + if (!key.IsCompressed()) return key.GetID(); + CTxDestination witdest = WitnessV0KeyHash(key.GetID()); + CScript witprog = GetScriptForDestination(witdest); + if (type == OUTPUT_TYPE_P2SH_SEGWIT) { + return CScriptID(witprog); + } else { + return witdest; + } + } + default: assert(false); + } +} + +std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key) +{ + CKeyID keyid = key.GetID(); + if (key.IsCompressed()) { + CTxDestination segwit = WitnessV0KeyHash(keyid); + CTxDestination p2sh = CScriptID(GetScriptForDestination(segwit)); + return std::vector<CTxDestination>{std::move(keyid), std::move(p2sh), std::move(segwit)}; + } else { + return std::vector<CTxDestination>{std::move(keyid)}; + } +} + +CTxDestination CWallet::AddAndGetDestinationForScript(const CScript& script, OutputType type) +{ + // Note that scripts over 520 bytes are not yet supported. + switch (type) { + case OUTPUT_TYPE_LEGACY: + return CScriptID(script); + case OUTPUT_TYPE_P2SH_SEGWIT: + case OUTPUT_TYPE_BECH32: { + WitnessV0ScriptHash hash; + CSHA256().Write(script.data(), script.size()).Finalize(hash.begin()); + CTxDestination witdest = hash; + CScript witprog = GetScriptForDestination(witdest); + // Check if the resulting program is solvable (i.e. doesn't use an uncompressed key) + if (!IsSolvable(*this, witprog)) return CScriptID(script); + // Add the redeemscript, so that P2WSH and P2SH-P2WSH outputs are recognized as ours. + AddCScript(witprog); + if (type == OUTPUT_TYPE_BECH32) { + return witdest; + } else { + return CScriptID(witprog); + } + } + default: assert(false); + } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c4af192f36..53a2c6b9d5 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -1,23 +1,23 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_WALLET_WALLET_H #define BITCOIN_WALLET_WALLET_H -#include "amount.h" -#include "policy/feerate.h" -#include "streams.h" -#include "tinyformat.h" -#include "ui_interface.h" -#include "utilstrencodings.h" -#include "validationinterface.h" -#include "script/ismine.h" -#include "script/sign.h" -#include "wallet/crypter.h" -#include "wallet/walletdb.h" -#include "wallet/rpcwallet.h" +#include <amount.h> +#include <policy/feerate.h> +#include <streams.h> +#include <tinyformat.h> +#include <ui_interface.h> +#include <utilstrencodings.h> +#include <validationinterface.h> +#include <script/ismine.h> +#include <script/sign.h> +#include <wallet/crypter.h> +#include <wallet/walletdb.h> +#include <wallet/rpcwallet.h> #include <algorithm> #include <atomic> @@ -99,6 +99,19 @@ enum WalletFeature FEATURE_LATEST = FEATURE_COMPRPUBKEY // HD is optional, use FEATURE_COMPRPUBKEY as latest version }; +enum OutputType +{ + OUTPUT_TYPE_NONE, + OUTPUT_TYPE_LEGACY, + OUTPUT_TYPE_P2SH_SEGWIT, + OUTPUT_TYPE_BECH32, + + OUTPUT_TYPE_DEFAULT = OUTPUT_TYPE_P2SH_SEGWIT +}; + +extern OutputType g_address_type; +extern OutputType g_change_type; + /** A key pool entry */ class CKeyPool @@ -214,10 +227,6 @@ public: Init(); } - /** Helper conversion operator to allow passing CMerkleTx where CTransaction is expected. - * TODO: adapt callers and remove this operator. */ - operator const CTransaction&() const { return *tx; } - void Init() { hashBlock = uint256(); @@ -252,8 +261,6 @@ public: int GetDepthInMainChain() const { const CBlockIndex *pindexRet; return GetDepthInMainChain(pindexRet); } bool IsInMainChain() const { const CBlockIndex *pindexRet; return GetDepthInMainChain(pindexRet) > 0; } int GetBlocksToMaturity() const; - /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ - bool AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state); bool hashUnset() const { return (hashBlock.IsNull() || hashBlock == ABANDON_HASH); } bool isAbandoned() const { return (hashBlock == ABANDON_HASH); } void setAbandoned() { hashBlock = ABANDON_HASH; } @@ -330,6 +337,7 @@ public: mutable bool fImmatureWatchCreditCached; mutable bool fAvailableWatchCreditCached; mutable bool fChangeCached; + mutable bool fInMempool; mutable CAmount nDebitCached; mutable CAmount nCreditCached; mutable CAmount nImmatureCreditCached; @@ -369,6 +377,7 @@ public: fImmatureWatchCreditCached = false; fAvailableWatchCreditCached = false; fChangeCached = false; + fInMempool = false; nDebitCached = 0; nCreditCached = 0; nImmatureCreditCached = 0; @@ -449,8 +458,8 @@ public: CAmount GetCredit(const isminefilter& filter) const; CAmount GetImmatureCredit(bool fUseCache=true) const; CAmount GetAvailableCredit(bool fUseCache=true) const; - CAmount GetImmatureWatchOnlyCredit(const bool& fUseCache=true) const; - CAmount GetAvailableWatchOnlyCredit(const bool& fUseCache=true) const; + CAmount GetImmatureWatchOnlyCredit(const bool fUseCache=true) const; + CAmount GetAvailableWatchOnlyCredit(const bool fUseCache=true) const; CAmount GetChange() const; void GetAmounts(std::list<COutputEntry>& listReceived, @@ -473,6 +482,9 @@ public: // RelayWalletTransaction may only be called if fBroadcastTransactions! bool RelayWalletTransaction(CConnman* connman); + /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ + bool AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state); + std::set<uint256> GetConflicts() const; }; @@ -722,6 +734,18 @@ private: std::unique_ptr<CWalletDBWrapper> dbw; + /** + * The following is used to keep track of how far behind the wallet is + * from the chain sync, and to allow clients to block on us being caught up. + * + * Note that this is *not* how far we've processed, we may need some rescan + * to have seen all transactions in the chain, but is only used to track + * live BlockConnected callbacks. + * + * Protected by cs_main (see BlockUntilSyncedToCurrentChain) + */ + const CBlockIndex* m_last_block_processed; + public: /* * Main wallet lock. @@ -750,9 +774,11 @@ public: void LoadKeyPool(int64_t nIndex, const CKeyPool &keypool); - // Map from Key ID (for regular keys) or Script ID (for watch-only keys) to - // key metadata. - std::map<CTxDestination, CKeyMetadata> mapKeyMetadata; + // Map from Key ID to key metadata. + std::map<CKeyID, CKeyMetadata> mapKeyMetadata; + + // Map from Script ID to key metadata (for watch-only keys). + std::map<CScriptID, CKeyMetadata> m_script_metadata; typedef std::map<unsigned int, CMasterKey> MasterKeyMap; MasterKeyMap mapMasterKeys; @@ -817,7 +843,7 @@ public: /** * populate vCoins with vector of available COutputs. */ - void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t& nMaximumCount = 0, const int& nMinDepth = 0, const int& nMaxDepth = 9999999) const; + void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0, const int nMinDepth = 0, const int nMaxDepth = 9999999) const; /** * Return list of available coins and locked coins grouped by non-change output address. @@ -863,7 +889,8 @@ public: //! Adds a key to the store, without saving it to disk (used by LoadWallet) bool LoadKey(const CKey& key, const CPubKey &pubkey) { return CCryptoKeyStore::AddKeyPubKey(key, pubkey); } //! Load metadata (used by LoadWallet) - bool LoadKeyMetadata(const CTxDestination& pubKey, const CKeyMetadata &metadata); + bool LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata); + bool LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata); bool LoadMinVersion(int nVersion) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; nWalletMaxVersion = std::max(nWalletMaxVersion, nVersion); return true; } void UpdateTimeFirstKey(int64_t nCreateTime); @@ -909,7 +936,7 @@ public: int64_t IncOrderPosNext(CWalletDB *pwalletdb = nullptr); DBErrors ReorderTransactions(); bool AccountMove(std::string strFrom, std::string strTo, CAmount nAmount, std::string strComment = ""); - bool GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bForceNew = false); + bool GetAccountDestination(CTxDestination &dest, std::string strAccount, bool bForceNew = false); void MarkDirty(); bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); @@ -919,7 +946,8 @@ public: void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override; bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate); int64_t RescanFromTime(int64_t startTime, bool update); - CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false); + CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop, bool fUpdate = false); + void TransactionRemovedFromMempool(const CTransactionRef &ptx) override; void ReacceptWalletTransactions(); void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override; // ResendWalletTransactionsBefore may only be called if fBroadcastTransactions! @@ -1106,6 +1134,34 @@ public: caller must ensure the current wallet version is correct before calling this function). */ bool SetHDMasterKey(const CPubKey& key); + + /** + * Blocks until the wallet state is up-to-date to /at least/ the current + * chain at the time this function is entered + * Obviously holding cs_main/cs_wallet when going into this call may cause + * deadlock + */ + void BlockUntilSyncedToCurrentChain(); + + /** + * Explicitly make the wallet learn the related scripts for outputs to the + * given key. This is purely to make the wallet file compatible with older + * software, as CBasicKeyStore automatically does this implicitly for all + * keys now. + */ + void LearnRelatedScripts(const CPubKey& key, OutputType); + + /** + * Same as LearnRelatedScripts, but when the OutputType is not known (and could + * be anything). + */ + void LearnAllRelatedScripts(const CPubKey& key); + + /** + * Get a destination of the requested type (if possible) to the specified script. + * This function will automatically add the necessary scripts to the wallet. + */ + CTxDestination AddAndGetDestinationForScript(const CScript& script, OutputType); }; /** A key allocated from the key pool. */ @@ -1195,4 +1251,16 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const ContainerType &coins return true; } +OutputType ParseOutputType(const std::string& str, OutputType default_type = OUTPUT_TYPE_DEFAULT); +const std::string& FormatOutputType(OutputType type); + +/** + * Get a destination of the requested type (if possible) to the specified key. + * The caller must make sure LearnRelatedScripts has been called beforehand. + */ +CTxDestination GetDestinationForKey(const CPubKey& key, OutputType); + +/** Get all destinations (potentially) supported by the wallet for the given key. */ +std::vector<CTxDestination> GetAllDestinationsForKey(const CPubKey& key); + #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index b7f873c1e4..dd6835a06f 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1,20 +1,20 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/walletdb.h" +#include <wallet/walletdb.h> -#include "base58.h" -#include "consensus/tx_verify.h" -#include "consensus/validation.h" -#include "fs.h" -#include "protocol.h" -#include "serialize.h" -#include "sync.h" -#include "util.h" -#include "utiltime.h" -#include "wallet/wallet.h" +#include <base58.h> +#include <consensus/tx_verify.h> +#include <consensus/validation.h> +#include <fs.h> +#include <protocol.h> +#include <serialize.h> +#include <sync.h> +#include <util.h> +#include <utiltime.h> +#include <wallet/wallet.h> #include <atomic> @@ -268,7 +268,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, CWalletTx wtx; ssValue >> wtx; CValidationState state; - if (!(CheckTransaction(wtx, state) && (wtx.GetHash() == hash) && state.IsValid())) + if (!(CheckTransaction(*wtx.tx, state) && (wtx.GetHash() == hash) && state.IsValid())) return false; // Undo serialize changes in 31600 @@ -423,27 +423,23 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } wss.fIsEncrypted = true; } - else if (strType == "keymeta" || strType == "watchmeta") + else if (strType == "keymeta") { - CTxDestination keyID; - if (strType == "keymeta") - { - CPubKey vchPubKey; - ssKey >> vchPubKey; - keyID = vchPubKey.GetID(); - } - else if (strType == "watchmeta") - { - CScript script; - ssKey >> script; - keyID = CScriptID(script); - } - + CPubKey vchPubKey; + ssKey >> vchPubKey; CKeyMetadata keyMeta; ssValue >> keyMeta; wss.nKeyMeta++; - - pwallet->LoadKeyMetadata(keyID, keyMeta); + pwallet->LoadKeyMetadata(vchPubKey.GetID(), keyMeta); + } + else if (strType == "watchmeta") + { + CScript script; + ssKey >> script; + CKeyMetadata keyMeta; + ssValue >> keyMeta; + wss.nKeyMeta++; + pwallet->LoadScriptMetadata(CScriptID(script), keyMeta); } else if (strType == "defaultkey") { @@ -630,7 +626,6 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx) { - bool fNoncriticalErrors = false; DBErrors result = DB_LOAD_OK; try { @@ -685,9 +680,6 @@ DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWal result = DB_CORRUPT; } - if (fNoncriticalErrors && result == DB_LOAD_OK) - result = DB_NONCRITICAL_ERROR; - return result; } @@ -814,14 +806,14 @@ bool CWalletDB::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDa return true; } -bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) +bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr) { - return CDB::VerifyEnvironment(walletFile, dataDir, errorStr); + return CDB::VerifyEnvironment(walletFile, walletDir, errorStr); } -bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr) +bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr) { - return CDB::VerifyDatabaseFile(walletFile, dataDir, warningStr, errorStr, CWalletDB::Recover); + return CDB::VerifyDatabaseFile(walletFile, walletDir, warningStr, errorStr, CWalletDB::Recover); } bool CWalletDB::WriteDestData(const std::string &address, const std::string &key, const std::string &value) diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 3a146179af..3691cfcb57 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -1,15 +1,15 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_WALLET_WALLETDB_H #define BITCOIN_WALLET_WALLETDB_H -#include "amount.h" -#include "primitives/transaction.h" -#include "wallet/db.h" -#include "key.h" +#include <amount.h> +#include <primitives/transaction.h> +#include <wallet/db.h> +#include <key.h> #include <list> #include <stdint.h> @@ -226,9 +226,9 @@ public: /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ static bool IsKeyType(const std::string& strType); /* verifies the database environment */ - static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& walletDir, std::string& errorStr); /* verifies the database file */ - static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr); + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& walletDir, std::string& warningStr, std::string& errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp new file mode 100644 index 0000000000..f15e5de1e2 --- /dev/null +++ b/src/wallet/walletutil.cpp @@ -0,0 +1,27 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <wallet/walletutil.h> + +fs::path GetWalletDir() +{ + fs::path path; + + if (gArgs.IsArgSet("-walletdir")) { + path = fs::system_complete(gArgs.GetArg("-walletdir", "")); + if (!fs::is_directory(path)) { + // If the path specified doesn't exist, we return the deliberately + // invalid empty string. + path = ""; + } + } else { + path = GetDataDir(); + // If a wallets directory exists, use that, otherwise default to GetDataDir + if (fs::is_directory(path / "wallets")) { + path /= "wallets"; + } + } + + return path; +} diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h new file mode 100644 index 0000000000..50ff736402 --- /dev/null +++ b/src/wallet/walletutil.h @@ -0,0 +1,14 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_UTIL_H +#define BITCOIN_WALLET_UTIL_H + +#include <chainparamsbase.h> +#include <util.h> + +//! Get the path of the wallet directory. +fs::path GetWalletDir(); + +#endif // BITCOIN_WALLET_UTIL_H diff --git a/src/warnings.cpp b/src/warnings.cpp index d4e33b701a..572c766600 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "sync.h" -#include "clientversion.h" -#include "util.h" -#include "warnings.h" +#include <sync.h> +#include <clientversion.h> +#include <util.h> +#include <warnings.h> CCriticalSection cs_warnings; std::string strMiscWarning; diff --git a/src/warnings.h b/src/warnings.h index e8e982c0e3..3d7ac5aab4 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2016 The Bitcoin Core developers +// Copyright (c) 2009-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/zmq/zmqabstractnotifier.cpp b/src/zmq/zmqabstractnotifier.cpp index 9f5cb3ba67..fc1ff6d031 100644 --- a/src/zmq/zmqabstractnotifier.cpp +++ b/src/zmq/zmqabstractnotifier.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "zmqabstractnotifier.h" -#include "util.h" +#include <zmq/zmqabstractnotifier.h> +#include <util.h> CZMQAbstractNotifier::~CZMQAbstractNotifier() diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h index 7828822149..7270ae203b 100644 --- a/src/zmq/zmqabstractnotifier.h +++ b/src/zmq/zmqabstractnotifier.h @@ -1,11 +1,11 @@ -// Copyright (c) 2015 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H #define BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H -#include "zmqconfig.h" +#include <zmq/zmqconfig.h> class CBlockIndex; class CZMQAbstractNotifier; diff --git a/src/zmq/zmqconfig.h b/src/zmq/zmqconfig.h index 610d7fbda4..1ba1262a83 100644 --- a/src/zmq/zmqconfig.h +++ b/src/zmq/zmqconfig.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014 The Bitcoin Core developers +// Copyright (c) 2014-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #define BITCOIN_ZMQ_ZMQCONFIG_H #if defined(HAVE_CONFIG_H) -#include "config/bitcoin-config.h" +#include <config/bitcoin-config.h> #endif #include <stdarg.h> @@ -16,8 +16,8 @@ #include <zmq.h> #endif -#include "primitives/block.h" -#include "primitives/transaction.h" +#include <primitives/block.h> +#include <primitives/transaction.h> void zmqError(const char *str); diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 9909395d84..68b425fa08 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "zmqnotificationinterface.h" -#include "zmqpublishnotifier.h" +#include <zmq/zmqnotificationinterface.h> +#include <zmq/zmqpublishnotifier.h> -#include "version.h" -#include "validation.h" -#include "streams.h" -#include "util.h" +#include <version.h> +#include <validation.h> +#include <streams.h> +#include <util.h> void zmqError(const char *str) { @@ -40,15 +40,15 @@ CZMQNotificationInterface* CZMQNotificationInterface::Create() factories["pubrawblock"] = CZMQAbstractNotifier::Create<CZMQPublishRawBlockNotifier>; factories["pubrawtx"] = CZMQAbstractNotifier::Create<CZMQPublishRawTransactionNotifier>; - for (std::map<std::string, CZMQNotifierFactory>::const_iterator i=factories.begin(); i!=factories.end(); ++i) + for (const auto& entry : factories) { - std::string arg("-zmq" + i->first); + std::string arg("-zmq" + entry.first); if (gArgs.IsArgSet(arg)) { - CZMQNotifierFactory factory = i->second; + CZMQNotifierFactory factory = entry.second; std::string address = gArgs.GetArg(arg, ""); CZMQAbstractNotifier *notifier = factory(); - notifier->SetType(i->first); + notifier->SetType(entry.first); notifier->SetAddress(address); notifiers.push_back(notifier); } diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index cb92216fa4..dee926ea5e 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -1,11 +1,11 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H #define BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H -#include "validationinterface.h" +#include <validationinterface.h> #include <string> #include <map> #include <list> diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index ab54e2bb8b..8c9acef257 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "chain.h" -#include "chainparams.h" -#include "streams.h" -#include "zmqpublishnotifier.h" -#include "validation.h" -#include "util.h" -#include "rpc/server.h" +#include <chain.h> +#include <chainparams.h> +#include <streams.h> +#include <zmq/zmqpublishnotifier.h> +#include <validation.h> +#include <util.h> +#include <rpc/server.h> static std::multimap<std::string, CZMQAbstractPublishNotifier*> mapPublishNotifiers; diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h index 1790fe5698..d53bba9971 100644 --- a/src/zmq/zmqpublishnotifier.h +++ b/src/zmq/zmqpublishnotifier.h @@ -1,11 +1,11 @@ -// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2015-2017 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_ZMQ_ZMQPUBLISHNOTIFIER_H #define BITCOIN_ZMQ_ZMQPUBLISHNOTIFIER_H -#include "zmqabstractnotifier.h" +#include <zmq/zmqabstractnotifier.h> class CBlockIndex; diff --git a/test/functional/README.md b/test/functional/README.md index 8d2cd02345..d6ce490ab3 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -17,7 +17,7 @@ don't have test cases for. #### Style guidelines -- Where possible, try to adhere to [PEP-8 guidelines]([https://www.python.org/dev/peps/pep-0008/) +- Where possible, try to adhere to [PEP-8 guidelines](https://www.python.org/dev/peps/pep-0008/) - Use a python linter like flake8 before submitting PRs to catch common style nits (eg trailing whitespace, unused imports, etc) - Avoid wildcard imports where possible @@ -77,12 +77,12 @@ wrappers for them, `msg_block`, `msg_tx`, etc). with the bitcoind(s) being tested (using python's asyncore package); the other implements the test logic. -- `NodeConn` is the class used to connect to a bitcoind. If you implement -a callback class that derives from `NodeConnCB` and pass that to the -`NodeConn` object, your code will receive the appropriate callbacks when -events of interest arrive. +- `P2PConnection` is the class used to connect to a bitcoind. `P2PInterface` +contains the higher level logic for processing P2P payloads and connecting to +the Bitcoin Core node application logic. For custom behaviour, subclass the +P2PInterface object and override the callback methods. -- Call `NetworkThread.start()` after all `NodeConn` objects are created to +- Call `network_thread_start()` after all `P2PInterface` objects are created to start the networking thread. (Continue with the test logic in your existing thread.) diff --git a/test/functional/abandonconflict.py b/test/functional/abandonconflict.py index e8dbc86469..14964438af 100755 --- a/test/functional/abandonconflict.py +++ b/test/functional/abandonconflict.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the abandontransaction RPC. diff --git a/test/functional/address_types.py b/test/functional/address_types.py new file mode 100755 index 0000000000..cb119c04b0 --- /dev/null +++ b/test/functional/address_types.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that the wallet can send and receive using all combinations of address types. + +There are 4 nodes-under-test: + - node0 uses legacy addresses + - node1 uses p2sh/segwit addresses + - node2 uses p2sh/segwit addresses and bech32 addresses for change + - node3 uses bech32 addresses + +node4 exists to generate new blocks. + +The script is a series of tests, iterating over the 4 nodes. In each iteration +of the test, one node sends: + - 10/101th of its balance to itself (using getrawchangeaddress for single key addresses) + - 20/101th to the next node + - 30/101th to the node after that + - 40/101th to the remaining node + - 1/101th remains as fee+change + +Iterate over each node for single key addresses, and then over each node for +multisig addresses. In a second iteration, the same is done, but with explicit address_type +parameters passed to getnewaddress and getrawchangeaddress. Node0 and node3 send to p2sh, +node 1 sends to bech32, and node2 sends to legacy. As every node sends coins after receiving, +this also verifies that spending coins sent to all these address types works.""" + +from decimal import Decimal +import itertools + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_greater_than, connect_nodes_bi, sync_blocks, sync_mempools + +class AddressTypeTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 5 + self.extra_args = [["-addresstype=legacy"], ["-addresstype=p2sh-segwit"], ["-addresstype=p2sh-segwit", "-changetype=bech32"], ["-addresstype=bech32"], []] + + def setup_network(self): + self.setup_nodes() + + # Fully mesh-connect nodes for faster mempool sync + for i, j in itertools.product(range(self.num_nodes), repeat=2): + if i > j: + connect_nodes_bi(self.nodes, i, j) + self.sync_all() + + def get_balances(self, confirmed=True): + """Return a list of confirmed or unconfirmed balances.""" + if confirmed: + return [self.nodes[i].getbalance() for i in range(4)] + else: + return [self.nodes[i].getunconfirmedbalance() for i in range(4)] + + def test_address(self, node, address, multisig, typ): + """Run sanity checks on an address.""" + info = self.nodes[node].validateaddress(address) + assert(info['isvalid']) + if not multisig and typ == 'legacy': + # P2PKH + assert(not info['isscript']) + assert(not info['iswitness']) + assert('pubkey' in info) + elif not multisig and typ == 'p2sh-segwit': + # P2SH-P2WPKH + assert(info['isscript']) + assert(not info['iswitness']) + assert_equal(info['script'], 'witness_v0_keyhash') + assert('pubkey' in info) + elif not multisig and typ == 'bech32': + # P2WPKH + assert(not info['isscript']) + assert(info['iswitness']) + assert_equal(info['witness_version'], 0) + assert_equal(len(info['witness_program']), 40) + assert('pubkey' in info) + elif typ == 'legacy': + # P2SH-multisig + assert(info['isscript']) + assert_equal(info['script'], 'multisig') + assert(not info['iswitness']) + assert('pubkeys' in info) + elif typ == 'p2sh-segwit': + # P2SH-P2WSH-multisig + assert(info['isscript']) + assert_equal(info['script'], 'witness_v0_scripthash') + assert(not info['iswitness']) + assert(info['embedded']['isscript']) + assert_equal(info['embedded']['script'], 'multisig') + assert(info['embedded']['iswitness']) + assert_equal(info['embedded']['witness_version'], 0) + assert_equal(len(info['embedded']['witness_program']), 64) + assert('pubkeys' in info['embedded']) + elif typ == 'bech32': + # P2WSH-multisig + assert(info['isscript']) + assert_equal(info['script'], 'multisig') + assert(info['iswitness']) + assert_equal(info['witness_version'], 0) + assert_equal(len(info['witness_program']), 64) + assert('pubkeys' in info) + else: + # Unknown type + assert(False) + + def run_test(self): + # Mine 101 blocks on node4 to bring nodes out of IBD and make sure that + # no coinbases are maturing for the nodes-under-test during the test + self.nodes[4].generate(101) + sync_blocks(self.nodes) + + uncompressed_1 = "0496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858ee" + uncompressed_2 = "047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77" + compressed_1 = "0296b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52" + compressed_2 = "037211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073" + + # addmultisigaddress with at least 1 uncompressed key should return a legacy address. + for node in range(4): + self.test_address(node, self.nodes[node].addmultisigaddress(2, [uncompressed_1, uncompressed_2]), True, 'legacy') + self.test_address(node, self.nodes[node].addmultisigaddress(2, [compressed_1, uncompressed_2]), True, 'legacy') + self.test_address(node, self.nodes[node].addmultisigaddress(2, [uncompressed_1, compressed_2]), True, 'legacy') + # addmultisigaddress with all compressed keys should return the appropriate address type (even when the keys are not ours). + self.test_address(0, self.nodes[0].addmultisigaddress(2, [compressed_1, compressed_2]), True, 'legacy') + self.test_address(1, self.nodes[1].addmultisigaddress(2, [compressed_1, compressed_2]), True, 'p2sh-segwit') + self.test_address(2, self.nodes[2].addmultisigaddress(2, [compressed_1, compressed_2]), True, 'p2sh-segwit') + self.test_address(3, self.nodes[3].addmultisigaddress(2, [compressed_1, compressed_2]), True, 'bech32') + + for explicit_type, multisig, from_node in itertools.product([False, True], [False, True], range(4)): + address_type = None + if explicit_type and not multisig: + if from_node == 1: + address_type = 'bech32' + elif from_node == 0 or from_node == 3: + address_type = 'p2sh-segwit' + else: + address_type = 'legacy' + self.log.info("Sending from node {} ({}) with{} multisig using {}".format(from_node, self.extra_args[from_node], "" if multisig else "out", "default" if address_type is None else address_type)) + old_balances = self.get_balances() + self.log.debug("Old balances are {}".format(old_balances)) + to_send = (old_balances[from_node] / 101).quantize(Decimal("0.00000001")) + sends = {} + + self.log.debug("Prepare sends") + for n, to_node in enumerate(range(from_node, from_node + 4)): + to_node %= 4 + change = False + if not multisig: + if from_node == to_node: + # When sending non-multisig to self, use getrawchangeaddress + address = self.nodes[to_node].getrawchangeaddress(address_type=address_type) + change = True + else: + address = self.nodes[to_node].getnewaddress(address_type=address_type) + else: + addr1 = self.nodes[to_node].getnewaddress() + addr2 = self.nodes[to_node].getnewaddress() + address = self.nodes[to_node].addmultisigaddress(2, [addr1, addr2]) + + # Do some sanity checking on the created address + if address_type is not None: + typ = address_type + elif to_node == 0: + typ = 'legacy' + elif to_node == 1 or (to_node == 2 and not change): + typ = 'p2sh-segwit' + else: + typ = 'bech32' + self.test_address(to_node, address, multisig, typ) + + # Output entry + sends[address] = to_send * 10 * (1 + n) + + self.log.debug("Sending: {}".format(sends)) + self.nodes[from_node].sendmany("", sends) + sync_mempools(self.nodes) + + unconf_balances = self.get_balances(False) + self.log.debug("Check unconfirmed balances: {}".format(unconf_balances)) + assert_equal(unconf_balances[from_node], 0) + for n, to_node in enumerate(range(from_node + 1, from_node + 4)): + to_node %= 4 + assert_equal(unconf_balances[to_node], to_send * 10 * (2 + n)) + + # node4 collects fee and block subsidy to keep accounting simple + self.nodes[4].generate(1) + sync_blocks(self.nodes) + + new_balances = self.get_balances() + self.log.debug("Check new balances: {}".format(new_balances)) + # We don't know what fee was set, so we can only check bounds on the balance of the sending node + assert_greater_than(new_balances[from_node], to_send * 10) + assert_greater_than(to_send * 11, new_balances[from_node]) + for n, to_node in enumerate(range(from_node + 1, from_node + 4)): + to_node %= 4 + assert_equal(new_balances[to_node], old_balances[to_node] + to_send * 10 * (2 + n)) + +if __name__ == '__main__': + AddressTypeTest().main() diff --git a/test/functional/assumevalid.py b/test/functional/assumevalid.py index 65685c48b7..5a09142412 100755 --- a/test/functional/assumevalid.py +++ b/test/functional/assumevalid.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for skipping signature validation on old blocks. @@ -38,16 +38,16 @@ from test_framework.mininode import (CBlockHeader, CTransaction, CTxIn, CTxOut, - NetworkThread, - NodeConn, - NodeConnCB, + network_thread_join, + network_thread_start, + P2PInterface, msg_block, msg_headers) from test_framework.script import (CScript, OP_TRUE) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (p2p_port, assert_equal) +from test_framework.util import assert_equal -class BaseNode(NodeConnCB): +class BaseNode(P2PInterface): def send_header_for_blocks(self, new_blocks): headers_message = msg_headers() headers_message.headers = [CBlockHeader(b) for b in new_blocks] @@ -65,13 +65,13 @@ class AssumeValidTest(BitcoinTestFramework): # signature so we can pass in the block hash as assumevalid. self.start_node(0) - def send_blocks_until_disconnected(self, node): + def send_blocks_until_disconnected(self, p2p_conn): """Keep sending blocks to the node until we're disconnected.""" for i in range(len(self.blocks)): - if not node.connection: + if p2p_conn.state != "connected": break try: - node.send_message(msg_block(self.blocks[i])) + p2p_conn.send_message(msg_block(self.blocks[i])) except IOError as e: assert str(e) == 'Not connected, no pushbuf' break @@ -97,13 +97,10 @@ class AssumeValidTest(BitcoinTestFramework): def run_test(self): # Connect to node0 - node0 = BaseNode() - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) - node0.add_connection(connections[0]) + p2p0 = self.nodes[0].add_p2p_connection(BaseNode()) - NetworkThread().start() # Start up network handling in another thread - node0.wait_for_verack() + network_thread_start() + self.nodes[0].p2p.wait_for_verack() # Build the blockchain self.tip = int(self.nodes[0].getbestblockhash(), 16) @@ -163,39 +160,44 @@ class AssumeValidTest(BitcoinTestFramework): self.block_time += 1 height += 1 + # We're adding new connections so terminate the network thread + self.nodes[0].disconnect_p2ps() + network_thread_join() + # Start node1 and node2 with assumevalid so they accept a block with a bad signature. self.start_node(1, extra_args=["-assumevalid=" + hex(block102.sha256)]) - node1 = BaseNode() # connects to node1 - connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], node1)) - node1.add_connection(connections[1]) - node1.wait_for_verack() - self.start_node(2, extra_args=["-assumevalid=" + hex(block102.sha256)]) - node2 = BaseNode() # connects to node2 - connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], node2)) - node2.add_connection(connections[2]) - node2.wait_for_verack() + + p2p0 = self.nodes[0].add_p2p_connection(BaseNode()) + p2p1 = self.nodes[1].add_p2p_connection(BaseNode()) + p2p2 = self.nodes[2].add_p2p_connection(BaseNode()) + + network_thread_start() + + p2p0.wait_for_verack() + p2p1.wait_for_verack() + p2p2.wait_for_verack() # send header lists to all three nodes - node0.send_header_for_blocks(self.blocks[0:2000]) - node0.send_header_for_blocks(self.blocks[2000:]) - node1.send_header_for_blocks(self.blocks[0:2000]) - node1.send_header_for_blocks(self.blocks[2000:]) - node2.send_header_for_blocks(self.blocks[0:200]) + p2p0.send_header_for_blocks(self.blocks[0:2000]) + p2p0.send_header_for_blocks(self.blocks[2000:]) + p2p1.send_header_for_blocks(self.blocks[0:2000]) + p2p1.send_header_for_blocks(self.blocks[2000:]) + p2p2.send_header_for_blocks(self.blocks[0:200]) # Send blocks to node0. Block 102 will be rejected. - self.send_blocks_until_disconnected(node0) + self.send_blocks_until_disconnected(p2p0) self.assert_blockchain_height(self.nodes[0], 101) # Send all blocks to node1. All blocks will be accepted. for i in range(2202): - node1.send_message(msg_block(self.blocks[i])) + p2p1.send_message(msg_block(self.blocks[i])) # Syncing 2200 blocks can take a while on slow systems. Give it plenty of time to sync. - node1.sync_with_ping(120) + p2p1.sync_with_ping(120) assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202) # Send blocks to node2. Block 102 will be rejected. - self.send_blocks_until_disconnected(node2) + self.send_blocks_until_disconnected(p2p2) self.assert_blockchain_height(self.nodes[2], 101) if __name__ == '__main__': diff --git a/test/functional/bip65-cltv-p2p.py b/test/functional/bip65-cltv-p2p.py index 2cd6df6e37..f62ae31654 100755 --- a/test/functional/bip65-cltv-p2p.py +++ b/test/functional/bip65-cltv-p2p.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP65 (CHECKLOCKTIMEVERIFY). @@ -66,15 +66,12 @@ class BIP65Test(BitcoinTestFramework): self.setup_clean_chain = True def run_test(self): - node0 = NodeConnCB() - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) - node0.add_connection(connections[0]) + self.nodes[0].add_p2p_connection(P2PInterface()) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() # wait_for_verack ensures that the P2P connection is fully up. - node0.wait_for_verack() + self.nodes[0].p2p.wait_for_verack() self.log.info("Mining %d blocks", CLTV_HEIGHT - 2) self.coinbase_blocks = self.nodes[0].generate(CLTV_HEIGHT - 2) @@ -95,7 +92,7 @@ class BIP65Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(self.nodes[0].getbestblockhash(), block.hash) self.log.info("Test that blocks must now be at least version 4") @@ -104,15 +101,15 @@ class BIP65Test(BitcoinTestFramework): block = create_block(tip, create_coinbase(CLTV_HEIGHT), block_time) block.nVersion = 3 block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - wait_until(lambda: "reject" in node0.last_message.keys(), lock=mininode_lock) + wait_until(lambda: "reject" in self.nodes[0].p2p.last_message.keys(), lock=mininode_lock) with mininode_lock: - assert_equal(node0.last_message["reject"].code, REJECT_OBSOLETE) - assert_equal(node0.last_message["reject"].reason, b'bad-version(0x00000003)') - assert_equal(node0.last_message["reject"].data, block.sha256) - del node0.last_message["reject"] + assert_equal(self.nodes[0].p2p.last_message["reject"].code, REJECT_OBSOLETE) + assert_equal(self.nodes[0].p2p.last_message["reject"].reason, b'bad-version(0x00000003)') + assert_equal(self.nodes[0].p2p.last_message["reject"].data, block.sha256) + del self.nodes[0].p2p.last_message["reject"] self.log.info("Test that invalid-according-to-cltv transactions cannot appear in a block") block.nVersion = 4 @@ -125,7 +122,7 @@ class BIP65Test(BitcoinTestFramework): # First we show that this tx is valid except for CLTV by getting it # accepted to the mempool (which we can achieve with # -promiscuousmempoolflags). - node0.send_and_ping(msg_tx(spendtx)) + self.nodes[0].p2p.send_and_ping(msg_tx(spendtx)) assert spendtx.hash in self.nodes[0].getrawmempool() # Now we verify that a block with this transaction is invalid. @@ -133,18 +130,18 @@ class BIP65Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - wait_until(lambda: "reject" in node0.last_message.keys(), lock=mininode_lock) + wait_until(lambda: "reject" in self.nodes[0].p2p.last_message.keys(), lock=mininode_lock) with mininode_lock: - assert node0.last_message["reject"].code in [REJECT_INVALID, REJECT_NONSTANDARD] - assert_equal(node0.last_message["reject"].data, block.sha256) - if node0.last_message["reject"].code == REJECT_INVALID: + assert self.nodes[0].p2p.last_message["reject"].code in [REJECT_INVALID, REJECT_NONSTANDARD] + assert_equal(self.nodes[0].p2p.last_message["reject"].data, block.sha256) + if self.nodes[0].p2p.last_message["reject"].code == REJECT_INVALID: # Generic rejection when a block is invalid - assert_equal(node0.last_message["reject"].reason, b'block-validation-failed') + assert_equal(self.nodes[0].p2p.last_message["reject"].reason, b'block-validation-failed') else: - assert b'Negative locktime' in node0.last_message["reject"].reason + assert b'Negative locktime' in self.nodes[0].p2p.last_message["reject"].reason self.log.info("Test that a version 4 block with a valid-according-to-CLTV transaction is accepted") spendtx = cltv_validate(self.nodes[0], spendtx, CLTV_HEIGHT - 1) @@ -155,7 +152,7 @@ class BIP65Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) diff --git a/test/functional/bip68-112-113-p2p.py b/test/functional/bip68-112-113-p2p.py index 7e6a4f4408..82aa0ff891 100755 --- a/test/functional/bip68-112-113-p2p.py +++ b/test/functional/bip68-112-113-p2p.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test activation of the first version bits soft fork. @@ -45,7 +45,7 @@ bip112tx_special - test negative argument to OP_CSV from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * -from test_framework.mininode import ToHex, CTransaction, NetworkThread +from test_framework.mininode import ToHex, CTransaction, network_thread_start from test_framework.blocktools import create_coinbase, create_block from test_framework.comptool import TestInstance, TestManager from test_framework.script import * @@ -95,12 +95,12 @@ class BIP68_112_113Test(ComparisonTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True - self.extra_args = [['-whitelist=127.0.0.1', '-blockversion=4']] + self.extra_args = [['-whitelist=127.0.0.1', '-blockversion=4', '-addresstype=legacy']] def run_test(self): test = TestManager(self, self.options.tmpdir) test.add_all_connections(self.nodes) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() test.run() def send_generic_input_tx(self, node, coinbases): diff --git a/test/functional/bip68-sequence.py b/test/functional/bip68-sequence.py index 5f8f21701f..94b13653b9 100755 --- a/test/functional/bip68-sequence.py +++ b/test/functional/bip68-sequence.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP68 implementation.""" @@ -362,9 +362,10 @@ class BIP68Test(BitcoinTestFramework): block.vtx.extend([tx1, tx2, tx3]) block.hashMerkleRoot = block.calc_merkle_root() block.rehash() + add_witness_commitment(block) block.solve() - self.nodes[0].submitblock(ToHex(block)) + self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) assert_equal(self.nodes[0].getbestblockhash(), block.hash) def activateCSV(self): diff --git a/test/functional/bip9-softforks.py b/test/functional/bip9-softforks.py index 904789301a..ae92e9f07c 100755 --- a/test/functional/bip9-softforks.py +++ b/test/functional/bip9-softforks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP 9 soft forks. @@ -22,7 +22,7 @@ import itertools from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * -from test_framework.mininode import CTransaction, NetworkThread +from test_framework.mininode import CTransaction, network_thread_start from test_framework.blocktools import create_coinbase, create_block from test_framework.comptool import TestInstance, TestManager from test_framework.script import CScript, OP_1NEGATE, OP_CHECKSEQUENCEVERIFY, OP_DROP @@ -36,7 +36,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): def run_test(self): self.test = TestManager(self, self.options.tmpdir) self.test.add_all_connections(self.nodes) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() self.test.run() def create_transaction(self, node, coinbase, to_address, amount): @@ -245,8 +245,8 @@ class BIP9SoftForksTest(ComparisonTestFramework): self.setup_chain() self.setup_network() self.test.add_all_connections(self.nodes) - NetworkThread().start() - self.test.test_nodes[0].wait_for_verack() + network_thread_start() + self.test.p2p_connections[0].wait_for_verack() def get_tests(self): for test in itertools.chain( diff --git a/test/functional/bipdersig-p2p.py b/test/functional/bipdersig-p2p.py index c620d3e155..3414571678 100755 --- a/test/functional/bipdersig-p2p.py +++ b/test/functional/bipdersig-p2p.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP66 (DER SIG). @@ -54,14 +54,12 @@ class BIP66Test(BitcoinTestFramework): self.setup_clean_chain = True def run_test(self): - node0 = NodeConnCB() - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) - node0.add_connection(connections[0]) - NetworkThread().start() # Start up network handling in another thread + self.nodes[0].add_p2p_connection(P2PInterface()) + + network_thread_start() # wait_for_verack ensures that the P2P connection is fully up. - node0.wait_for_verack() + self.nodes[0].p2p.wait_for_verack() self.log.info("Mining %d blocks", DERSIG_HEIGHT - 2) self.coinbase_blocks = self.nodes[0].generate(DERSIG_HEIGHT - 2) @@ -83,7 +81,7 @@ class BIP66Test(BitcoinTestFramework): block.rehash() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(self.nodes[0].getbestblockhash(), block.hash) self.log.info("Test that blocks must now be at least version 3") @@ -93,15 +91,15 @@ class BIP66Test(BitcoinTestFramework): block.nVersion = 2 block.rehash() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - wait_until(lambda: "reject" in node0.last_message.keys(), lock=mininode_lock) + wait_until(lambda: "reject" in self.nodes[0].p2p.last_message.keys(), lock=mininode_lock) with mininode_lock: - assert_equal(node0.last_message["reject"].code, REJECT_OBSOLETE) - assert_equal(node0.last_message["reject"].reason, b'bad-version(0x00000002)') - assert_equal(node0.last_message["reject"].data, block.sha256) - del node0.last_message["reject"] + assert_equal(self.nodes[0].p2p.last_message["reject"].code, REJECT_OBSOLETE) + assert_equal(self.nodes[0].p2p.last_message["reject"].reason, b'bad-version(0x00000002)') + assert_equal(self.nodes[0].p2p.last_message["reject"].data, block.sha256) + del self.nodes[0].p2p.last_message["reject"] self.log.info("Test that transactions with non-DER signatures cannot appear in a block") block.nVersion = 3 @@ -114,7 +112,7 @@ class BIP66Test(BitcoinTestFramework): # First we show that this tx is valid except for DERSIG by getting it # accepted to the mempool (which we can achieve with # -promiscuousmempoolflags). - node0.send_and_ping(msg_tx(spendtx)) + self.nodes[0].p2p.send_and_ping(msg_tx(spendtx)) assert spendtx.hash in self.nodes[0].getrawmempool() # Now we verify that a block with this transaction is invalid. @@ -123,23 +121,23 @@ class BIP66Test(BitcoinTestFramework): block.rehash() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) - wait_until(lambda: "reject" in node0.last_message.keys(), lock=mininode_lock) + wait_until(lambda: "reject" in self.nodes[0].p2p.last_message.keys(), lock=mininode_lock) with mininode_lock: # We can receive different reject messages depending on whether # bitcoind is running with multiple script check threads. If script # check threads are not in use, then transaction script validation # happens sequentially, and bitcoind produces more specific reject # reasons. - assert node0.last_message["reject"].code in [REJECT_INVALID, REJECT_NONSTANDARD] - assert_equal(node0.last_message["reject"].data, block.sha256) - if node0.last_message["reject"].code == REJECT_INVALID: + assert self.nodes[0].p2p.last_message["reject"].code in [REJECT_INVALID, REJECT_NONSTANDARD] + assert_equal(self.nodes[0].p2p.last_message["reject"].data, block.sha256) + if self.nodes[0].p2p.last_message["reject"].code == REJECT_INVALID: # Generic rejection when a block is invalid - assert_equal(node0.last_message["reject"].reason, b'block-validation-failed') + assert_equal(self.nodes[0].p2p.last_message["reject"].reason, b'block-validation-failed') else: - assert b'Non-canonical DER signature' in node0.last_message["reject"].reason + assert b'Non-canonical DER signature' in self.nodes[0].p2p.last_message["reject"].reason self.log.info("Test that a version 3 block with a DERSIG-compliant transaction is accepted") block.vtx[1] = create_transaction(self.nodes[0], @@ -148,7 +146,7 @@ class BIP66Test(BitcoinTestFramework): block.rehash() block.solve() - node0.send_and_ping(msg_block(block)) + self.nodes[0].p2p.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) if __name__ == '__main__': diff --git a/test/functional/bitcoin_cli.py b/test/functional/bitcoin_cli.py index 996cbb8a12..d1cd3b3620 100755 --- a/test/functional/bitcoin_cli.py +++ b/test/functional/bitcoin_cli.py @@ -35,8 +35,11 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(["foo", "bar"], self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input=password + "\nfoo\nbar").echo()) assert_raises_process_error(1, "incorrect rpcuser or rpcpassword", self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input="foo").echo) + self.log.info("Make sure that -getinfo with arguments fails") + assert_raises_process_error(1, "-getinfo takes no arguments", self.nodes[0].cli('-getinfo').help) + self.log.info("Compare responses from `bitcoin-cli -getinfo` and the RPCs data is retrieved from.") - cli_get_info = self.nodes[0].cli('-getinfo').help() + cli_get_info = self.nodes[0].cli().send_cli('-getinfo') wallet_info = self.nodes[0].getwalletinfo() network_info = self.nodes[0].getnetworkinfo() blockchain_info = self.nodes[0].getblockchaininfo() diff --git a/test/functional/blockchain.py b/test/functional/blockchain.py index 4576cb036a..11acff4be1 100755 --- a/test/functional/blockchain.py +++ b/test/functional/blockchain.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPCs related to blockchainstate. Test the following RPCs: + - getblockchaininfo - gettxoutsetinfo - getdifficulty - getbestblockhash @@ -58,6 +59,7 @@ class BlockchainTest(BitcoinTestFramework): 'chainwork', 'difficulty', 'headers', + 'initialblockdownload', 'mediantime', 'pruned', 'size_on_disk', diff --git a/test/functional/bumpfee.py b/test/functional/bumpfee.py index 44fa7b0feb..1e5620736b 100755 --- a/test/functional/bumpfee.py +++ b/test/functional/bumpfee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the bumpfee RPC. @@ -194,7 +194,7 @@ def test_settxfee(rbf_node, dest_address): requested_feerate = Decimal("0.00025000") rbf_node.settxfee(requested_feerate) bumped_tx = rbf_node.bumpfee(rbfid) - actual_feerate = bumped_tx["fee"] * 1000 / rbf_node.getrawtransaction(bumped_tx["txid"], True)["size"] + actual_feerate = bumped_tx["fee"] * 1000 / rbf_node.getrawtransaction(bumped_tx["txid"], True)["vsize"] # Assert that the difference between the requested feerate and the actual # feerate of the bumped transaction is small. assert_greater_than(Decimal("0.00001000"), abs(requested_feerate - actual_feerate)) @@ -290,6 +290,7 @@ def submit_block_with_tx(node, tx): block.vtx.append(ctx) block.rehash() block.hashMerkleRoot = block.calc_merkle_root() + blocktools.add_witness_commitment(block) block.solve() node.submitblock(bytes_to_hex_str(block.serialize(True))) return block diff --git a/test/functional/conf_args.py b/test/functional/conf_args.py new file mode 100755 index 0000000000..61abba8082 --- /dev/null +++ b/test/functional/conf_args.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test various command line arguments and configuration file parameters.""" + +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import get_datadir_path + +class ConfArgsTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def run_test(self): + self.stop_node(0) + # Remove the -datadir argument so it doesn't override the config file + self.nodes[0].args = [arg for arg in self.nodes[0].args if not arg.startswith("-datadir")] + + default_data_dir = get_datadir_path(self.options.tmpdir, 0) + new_data_dir = os.path.join(default_data_dir, 'newdatadir') + new_data_dir_2 = os.path.join(default_data_dir, 'newdatadir2') + + # Check that using -datadir argument on non-existent directory fails + self.nodes[0].datadir = new_data_dir + self.assert_start_raises_init_error(0, ['-datadir='+new_data_dir], 'Error: Specified data directory "' + new_data_dir + '" does not exist.') + + # Check that using non-existent datadir in conf file fails + conf_file = os.path.join(default_data_dir, "bitcoin.conf") + with open(conf_file, 'a', encoding='utf8') as f: + f.write("datadir=" + new_data_dir + "\n") + self.assert_start_raises_init_error(0, ['-conf='+conf_file], 'Error reading configuration file: specified data directory "' + new_data_dir + '" does not exist.') + + # Create the directory and ensure the config file now works + os.mkdir(new_data_dir) + self.start_node(0, ['-conf='+conf_file, '-wallet=w1']) + self.stop_node(0) + assert os.path.isfile(os.path.join(new_data_dir, 'regtest', 'wallets', 'w1')) + + # Ensure command line argument overrides datadir in conf + os.mkdir(new_data_dir_2) + self.nodes[0].datadir = new_data_dir_2 + self.start_node(0, ['-datadir='+new_data_dir_2, '-conf='+conf_file, '-wallet=w2']) + assert os.path.isfile(os.path.join(new_data_dir_2, 'regtest', 'wallets', 'w2')) + +if __name__ == '__main__': + ConfArgsTest().main() diff --git a/test/functional/create_cache.py b/test/functional/create_cache.py index 7d4d1a529b..9665c50a92 100755 --- a/test/functional/create_cache.py +++ b/test/functional/create_cache.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Create a blockchain cache. @@ -16,6 +16,7 @@ class CreateCache(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 0 + self.supports_cli = True def setup_network(self): pass diff --git a/test/functional/decodescript.py b/test/functional/decodescript.py index 6611da8831..1ffc570437 100755 --- a/test/functional/decodescript.py +++ b/test/functional/decodescript.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test decoding scripts via decodescript RPC command.""" diff --git a/test/functional/disablewallet.py b/test/functional/disablewallet.py index c75ef9b9f1..b0627d88ac 100755 --- a/test/functional/disablewallet.py +++ b/test/functional/disablewallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test a node with the -disablewallet option. diff --git a/test/functional/disconnect_ban.py b/test/functional/disconnect_ban.py index 59655d37fb..c6067befb2 100755 --- a/test/functional/disconnect_ban.py +++ b/test/functional/disconnect_ban.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node disconnect and ban behavior""" diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 87d73ad14a..12be685ecf 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -17,30 +17,29 @@ from collections import defaultdict from test_framework.blocktools import (create_block, create_coinbase) from test_framework.mininode import ( CInv, - NetworkThread, - NodeConn, - NodeConnCB, + P2PInterface, mininode_lock, msg_block, msg_getdata, + network_thread_join, + network_thread_start, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, connect_nodes, - p2p_port, wait_until, ) -# NodeConnCB is a class containing callbacks to be executed when a P2P -# message is received from the node-under-test. Subclass NodeConnCB and +# P2PInterface is a class containing callbacks to be executed when a P2P +# message is received from the node-under-test. Subclass P2PInterface and # override the on_*() methods if you need custom behaviour. -class BaseNode(NodeConnCB): +class BaseNode(P2PInterface): def __init__(self): - """Initialize the NodeConnCB + """Initialize the P2PInterface Used to inialize custom properties for the Node that aren't - included by default in the base class. Be aware that the NodeConnCB + included by default in the base class. Be aware that the P2PInterface base class already stores a counter for each P2P message type and the last received message of each type, which should be sufficient for the needs of most tests. @@ -51,14 +50,14 @@ class BaseNode(NodeConnCB): # Stores a dictionary of all blocks received self.block_receive_map = defaultdict(int) - def on_block(self, conn, message): + def on_block(self, message): """Override the standard on_block callback Store the hash of a received block in the dictionary.""" message.block.calc_sha256() self.block_receive_map[message.block.sha256] += 1 - def on_inv(self, conn, message): + def on_inv(self, message): """Override the standard on_inv callback""" pass @@ -133,17 +132,14 @@ class ExampleTest(BitcoinTestFramework): def run_test(self): """Main test logic""" - # Create a P2P connection to one of the nodes - node0 = BaseNode() - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], node0)) - node0.add_connection(connections[0]) + # Create P2P connections to two of the nodes + self.nodes[0].add_p2p_connection(BaseNode()) # Start up network handling in another thread. This needs to be called # after the P2P connections have been created. - NetworkThread().start() + network_thread_start() # wait_for_verack ensures that the P2P connection is fully up. - node0.wait_for_verack() + self.nodes[0].p2p.wait_for_verack() # Generating a block on one of the nodes will get us out of IBD blocks = [int(self.nodes[0].generate(nblocks=1)[0], 16)] @@ -179,8 +175,8 @@ class ExampleTest(BitcoinTestFramework): block = create_block(self.tip, create_coinbase(height), self.block_time) block.solve() block_message = msg_block(block) - # Send message is used to send a P2P message to the node over our NodeConn connection - node0.send_message(block_message) + # Send message is used to send a P2P message to the node over our P2PInterface + self.nodes[0].p2p.send_message(block_message) self.tip = block.sha256 blocks.append(self.tip) self.block_time += 1 @@ -193,28 +189,33 @@ class ExampleTest(BitcoinTestFramework): connect_nodes(self.nodes[1], 2) self.log.info("Add P2P connection to node2") - node2 = BaseNode() - connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], node2)) - node2.add_connection(connections[1]) - node2.wait_for_verack() + # We can't add additional P2P connections once the network thread has started. Disconnect the connection + # to node0, wait for the network thread to terminate, then connect to node2. This is specific to + # the current implementation of the network thread and may be improved in future. + self.nodes[0].disconnect_p2ps() + network_thread_join() + + self.nodes[2].add_p2p_connection(BaseNode()) + network_thread_start() + self.nodes[2].p2p.wait_for_verack() self.log.info("Wait for node2 reach current tip. Test that it has propagated all the blocks to us") getdata_request = msg_getdata() for block in blocks: getdata_request.inv.append(CInv(2, block)) - node2.send_message(getdata_request) + self.nodes[2].p2p.send_message(getdata_request) # wait_until() will loop until a predicate condition is met. Use it to test properties of the - # NodeConnCB objects. - wait_until(lambda: sorted(blocks) == sorted(list(node2.block_receive_map.keys())), timeout=5, lock=mininode_lock) + # P2PInterface objects. + wait_until(lambda: sorted(blocks) == sorted(list(self.nodes[2].p2p.block_receive_map.keys())), timeout=5, lock=mininode_lock) self.log.info("Check that each block was received only once") - # The network thread uses a global lock on data access to the NodeConn objects when sending and receiving - # messages. The test thread should acquire the global lock before accessing any NodeConn data to avoid locking + # The network thread uses a global lock on data access to the P2PConnection objects when sending and receiving + # messages. The test thread should acquire the global lock before accessing any P2PConnection data to avoid locking # and synchronization issues. Note wait_until() acquires this global lock when testing the predicate. with mininode_lock: - for block in node2.block_receive_map.values(): + for block in self.nodes[2].p2p.block_receive_map.values(): assert_equal(block, 1) if __name__ == '__main__': diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py new file mode 100755 index 0000000000..da4e7b0398 --- /dev/null +++ b/test/functional/feature_logging.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test debug logging.""" + +import os + +from test_framework.test_framework import BitcoinTestFramework + +class LoggingTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + # test default log file name + assert os.path.isfile(os.path.join(self.nodes[0].datadir, "regtest", "debug.log")) + + # test alternative log file name in datadir + self.restart_node(0, ["-debuglogfile=foo.log"]) + assert os.path.isfile(os.path.join(self.nodes[0].datadir, "regtest", "foo.log")) + + # test alternative log file name outside datadir + tempname = os.path.join(self.options.tmpdir, "foo.log") + self.restart_node(0, ["-debuglogfile=%s" % tempname]) + assert os.path.isfile(tempname) + + # check that invalid log (relative) will cause error + invdir = os.path.join(self.nodes[0].datadir, "regtest", "foo") + invalidname = os.path.join("foo", "foo.log") + self.stop_node(0) + self.assert_start_raises_init_error(0, ["-debuglogfile=%s" % (invalidname)], + "Error: Could not open debug log file") + assert not os.path.isfile(os.path.join(invdir, "foo.log")) + + # check that invalid log (relative) works after path exists + self.stop_node(0) + os.mkdir(invdir) + self.start_node(0, ["-debuglogfile=%s" % (invalidname)]) + assert os.path.isfile(os.path.join(invdir, "foo.log")) + + # check that invalid log (absolute) will cause error + self.stop_node(0) + invdir = os.path.join(self.options.tmpdir, "foo") + invalidname = os.path.join(invdir, "foo.log") + self.assert_start_raises_init_error(0, ["-debuglogfile=%s" % invalidname], + "Error: Could not open debug log file") + assert not os.path.isfile(os.path.join(invdir, "foo.log")) + + # check that invalid log (absolute) works after path exists + self.stop_node(0) + os.mkdir(invdir) + self.start_node(0, ["-debuglogfile=%s" % (invalidname)]) + assert os.path.isfile(os.path.join(invdir, "foo.log")) + + +if __name__ == '__main__': + LoggingTest().main() diff --git a/test/functional/forknotify.py b/test/functional/forknotify.py deleted file mode 100755 index afcad1f9cc..0000000000 --- a/test/functional/forknotify.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test the -alertnotify option.""" -import os -import time - -from test_framework.test_framework import BitcoinTestFramework - -class ForkNotifyTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 2 - - def setup_network(self): - self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") - with open(self.alert_filename, 'w', encoding='utf8'): - pass # Just open then close to create zero-length file - self.extra_args = [["-blockversion=2", "-alertnotify=echo %s >> \"" + self.alert_filename + "\""], - ["-blockversion=211"]] - super().setup_network() - - def run_test(self): - # Mine 51 up-version blocks - self.nodes[1].generate(51) - self.sync_all() - # -alertnotify should trigger on the 51'st, - # but mine and sync another to give - # -alertnotify time to write - self.nodes[1].generate(1) - self.sync_all() - - # Give bitcoind 10 seconds to write the alert notification - timeout = 10.0 - while timeout > 0: - if os.path.exists(self.alert_filename) and os.path.getsize(self.alert_filename): - break - time.sleep(0.1) - timeout -= 0.1 - else: - assert False, "-alertnotify did not warn of up-version blocks" - - with open(self.alert_filename, 'r', encoding='utf8') as f: - alert_text = f.read() - - # Mine more up-version blocks, should not get more alerts: - self.nodes[1].generate(1) - self.sync_all() - self.nodes[1].generate(1) - self.sync_all() - - with open(self.alert_filename, 'r', encoding='utf8') as f: - alert_text2 = f.read() - - if alert_text != alert_text2: - raise AssertionError("-alertnotify excessive warning of up-version blocks") - -if __name__ == '__main__': - ForkNotifyTest().main() diff --git a/test/functional/fundrawtransaction.py b/test/functional/fundrawtransaction.py index d446f56d0e..4739e73c39 100755 --- a/test/functional/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the fundrawtransaction RPC.""" diff --git a/test/functional/getblocktemplate_longpoll.py b/test/functional/getblocktemplate_longpoll.py index 89768bd2fb..252ff4dbff 100755 --- a/test/functional/getblocktemplate_longpoll.py +++ b/test/functional/getblocktemplate_longpoll.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test longpolling with getblocktemplate.""" diff --git a/test/functional/getchaintips.py b/test/functional/getchaintips.py index 21b67bfc64..277930bb1a 100755 --- a/test/functional/getchaintips.py +++ b/test/functional/getchaintips.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the getchaintips RPC. diff --git a/test/functional/httpbasics.py b/test/functional/httpbasics.py index c7682cb49d..cd6d744545 100755 --- a/test/functional/httpbasics.py +++ b/test/functional/httpbasics.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC HTTP basics.""" diff --git a/test/functional/import-rescan.py b/test/functional/import-rescan.py index 6807fa6696..d193a99d5b 100755 --- a/test/functional/import-rescan.py +++ b/test/functional/import-rescan.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet import RPCs. @@ -119,7 +119,7 @@ class ImportRescanTest(BitcoinTestFramework): self.num_nodes = 2 + len(IMPORT_NODES) def setup_network(self): - extra_args = [[] for _ in range(self.num_nodes)] + extra_args = [["-addresstype=legacy"] for _ in range(self.num_nodes)] for i, import_node in enumerate(IMPORT_NODES, 2): if import_node.prune: extra_args[i] += ["-prune=1"] diff --git a/test/functional/importmulti.py b/test/functional/importmulti.py index c1a42870ec..ab4ca48e48 100755 --- a/test/functional/importmulti.py +++ b/test/functional/importmulti.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importmulti RPC.""" @@ -9,6 +9,7 @@ from test_framework.util import * class ImportMultiTest (BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + self.extra_args = [["-addresstype=legacy"], ["-addresstype=legacy"]] self.setup_clean_chain = True def setup_network(self): @@ -160,6 +161,18 @@ class ImportMultiTest (BitcoinTestFramework): assert_equal(address_assert['ismine'], True) assert_equal(address_assert['timestamp'], timestamp) + self.log.info("Should not import an address with private key if is already imported") + result = self.nodes[1].importmulti([{ + "scriptPubKey": { + "address": address['address'] + }, + "timestamp": "now", + "keys": [ self.nodes[0].dumpprivkey(address['address']) ] + }]) + assert_equal(result[0]['success'], False) + assert_equal(result[0]['error']['code'], -4) + assert_equal(result[0]['error']['message'], 'The wallet already contains the private key for this address or script') + # Address + Private key + watchonly self.log.info("Should not import an address with private key and with watchonly") address = self.nodes[0].validateaddress(self.nodes[0].getnewaddress()) diff --git a/test/functional/importprunedfunds.py b/test/functional/importprunedfunds.py index 068052409a..6b2919b5ae 100755 --- a/test/functional/importprunedfunds.py +++ b/test/functional/importprunedfunds.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importprunedfunds and removeprunedfunds RPCs.""" diff --git a/test/functional/invalidateblock.py b/test/functional/invalidateblock.py index dd3daf1e07..b037c2431d 100755 --- a/test/functional/invalidateblock.py +++ b/test/functional/invalidateblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the invalidateblock RPC.""" diff --git a/test/functional/invalidblockrequest.py b/test/functional/invalidblockrequest.py index 9f44b44927..edcade63c1 100755 --- a/test/functional/invalidblockrequest.py +++ b/test/functional/invalidblockrequest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid blocks. @@ -15,6 +15,7 @@ from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * from test_framework.comptool import TestManager, TestInstance, RejectResult from test_framework.blocktools import * +from test_framework.mininode import network_thread_start import copy import time @@ -32,7 +33,7 @@ class InvalidBlockRequestTest(ComparisonTestFramework): test.add_all_connections(self.nodes) self.tip = None self.block_time = None - NetworkThread().start() # Start up network handling in another thread + network_thread_start() test.run() def get_tests(self): diff --git a/test/functional/invalidtxrequest.py b/test/functional/invalidtxrequest.py index a22bd8f8cd..9c1100e070 100755 --- a/test/functional/invalidtxrequest.py +++ b/test/functional/invalidtxrequest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid transactions. @@ -28,7 +28,7 @@ class InvalidTxRequestTest(ComparisonTestFramework): test.add_all_connections(self.nodes) self.tip = None self.block_time = None - NetworkThread().start() # Start up network handling in another thread + network_thread_start() test.run() def get_tests(self): diff --git a/test/functional/keypool-topup.py b/test/functional/keypool-topup.py index 160a0f7ae5..e7af3c3987 100755 --- a/test/functional/keypool-topup.py +++ b/test/functional/keypool-topup.py @@ -33,7 +33,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.stop_node(1) - shutil.copyfile(self.tmpdir + "/node1/regtest/wallet.dat", self.tmpdir + "/wallet.bak") + shutil.copyfile(self.tmpdir + "/node1/regtest/wallets/wallet.dat", self.tmpdir + "/wallet.bak") self.start_node(1, self.extra_args[1]) connect_nodes_bi(self.nodes, 0, 1) @@ -56,7 +56,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.stop_node(1) - shutil.copyfile(self.tmpdir + "/wallet.bak", self.tmpdir + "/node1/regtest/wallet.dat") + shutil.copyfile(self.tmpdir + "/wallet.bak", self.tmpdir + "/node1/regtest/wallets/wallet.dat") self.log.info("Verify keypool is restored and balance is correct") diff --git a/test/functional/keypool.py b/test/functional/keypool.py index f2701c36bd..45a5eed8ec 100755 --- a/test/functional/keypool.py +++ b/test/functional/keypool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet keypool and interaction with wallet encryption/locking.""" diff --git a/test/functional/listsinceblock.py b/test/functional/listsinceblock.py index 6f428388ec..67e7744bf8 100755 --- a/test/functional/listsinceblock.py +++ b/test/functional/listsinceblock.py @@ -5,7 +5,7 @@ """Test the listsincelast RPC.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal +from test_framework.util import assert_equal, assert_array_result, assert_raises_rpc_error class ListSinceBlockTest (BitcoinTestFramework): def set_test_params(self): @@ -16,10 +16,43 @@ class ListSinceBlockTest (BitcoinTestFramework): self.nodes[2].generate(101) self.sync_all() + self.test_no_blockhash() + self.test_invalid_blockhash() self.test_reorg() self.test_double_spend() self.test_double_send() + def test_no_blockhash(self): + txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) + blockhash, = self.nodes[2].generate(1) + self.sync_all() + + txs = self.nodes[0].listtransactions() + assert_array_result(txs, {"txid": txid}, { + "category": "receive", + "amount": 1, + "blockhash": blockhash, + "confirmations": 1, + }) + assert_equal( + self.nodes[0].listsinceblock(), + {"lastblock": blockhash, + "removed": [], + "transactions": txs}) + assert_equal( + self.nodes[0].listsinceblock(""), + {"lastblock": blockhash, + "removed": [], + "transactions": txs}) + + def test_invalid_blockhash(self): + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].listsinceblock, + "42759cde25462784395a337460bde75f58e73d3f08bd31fdc3507cbac856a2c4") + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].listsinceblock, + "0000000000000000000000000000000000000000000000000000000000000000") + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].listsinceblock, + "invalid-hex") + def test_reorg(self): ''' `listsinceblock` did not behave correctly when handed a block that was diff --git a/test/functional/listtransactions.py b/test/functional/listtransactions.py index e4522cc3b5..cbed065928 100755 --- a/test/functional/listtransactions.py +++ b/test/functional/listtransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listtransactions API.""" diff --git a/test/functional/maxuploadtarget.py b/test/functional/maxuploadtarget.py index 1f402798e7..45336ee801 100755 --- a/test/functional/maxuploadtarget.py +++ b/test/functional/maxuploadtarget.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of -maxuploadtarget. @@ -17,15 +17,15 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -class TestNode(NodeConnCB): +class TestNode(P2PInterface): def __init__(self): super().__init__() self.block_receive_map = defaultdict(int) - def on_inv(self, conn, message): + def on_inv(self, message): pass - def on_block(self, conn, message): + def on_block(self, message): message.block.calc_sha256() self.block_receive_map[message.block.sha256] += 1 @@ -49,19 +49,17 @@ class MaxUploadTest(BitcoinTestFramework): # Generate some old blocks self.nodes[0].generate(130) - # test_nodes[0] will only request old blocks - # test_nodes[1] will only request new blocks - # test_nodes[2] will test resetting the counters - test_nodes = [] - connections = [] + # p2p_conns[0] will only request old blocks + # p2p_conns[1] will only request new blocks + # p2p_conns[2] will test resetting the counters + p2p_conns = [] - for i in range(3): - test_nodes.append(TestNode()) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[i])) - test_nodes[i].add_connection(connections[i]) + for _ in range(3): + p2p_conns.append(self.nodes[0].add_p2p_connection(TestNode())) - NetworkThread().start() # Start up network handling in another thread - [x.wait_for_verack() for x in test_nodes] + network_thread_start() + for p2pc in p2p_conns: + p2pc.wait_for_verack() # Test logic begins here @@ -83,7 +81,7 @@ class MaxUploadTest(BitcoinTestFramework): big_new_block = self.nodes[0].getbestblockhash() big_new_block = int(big_new_block, 16) - # test_nodes[0] will test what happens if we just keep requesting the + # p2p_conns[0] will test what happens if we just keep requesting the # the same big old block too many times (expect: disconnect) getdata_request = msg_getdata() @@ -97,34 +95,34 @@ class MaxUploadTest(BitcoinTestFramework): # 576MB will be reserved for relaying new blocks, so expect this to # succeed for ~235 tries. for i in range(success_count): - test_nodes[0].send_message(getdata_request) - test_nodes[0].sync_with_ping() - assert_equal(test_nodes[0].block_receive_map[big_old_block], i+1) + p2p_conns[0].send_message(getdata_request) + p2p_conns[0].sync_with_ping() + assert_equal(p2p_conns[0].block_receive_map[big_old_block], i+1) assert_equal(len(self.nodes[0].getpeerinfo()), 3) # At most a couple more tries should succeed (depending on how long # the test has been running so far). for i in range(3): - test_nodes[0].send_message(getdata_request) - test_nodes[0].wait_for_disconnect() + p2p_conns[0].send_message(getdata_request) + p2p_conns[0].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 2) self.log.info("Peer 0 disconnected after downloading old block too many times") - # Requesting the current block on test_nodes[1] should succeed indefinitely, + # Requesting the current block on p2p_conns[1] should succeed indefinitely, # even when over the max upload target. # We'll try 800 times getdata_request.inv = [CInv(2, big_new_block)] for i in range(800): - test_nodes[1].send_message(getdata_request) - test_nodes[1].sync_with_ping() - assert_equal(test_nodes[1].block_receive_map[big_new_block], i+1) + p2p_conns[1].send_message(getdata_request) + p2p_conns[1].sync_with_ping() + assert_equal(p2p_conns[1].block_receive_map[big_new_block], i+1) self.log.info("Peer 1 able to repeatedly download new block") - # But if test_nodes[1] tries for an old block, it gets disconnected too. + # But if p2p_conns[1] tries for an old block, it gets disconnected too. getdata_request.inv = [CInv(2, big_old_block)] - test_nodes[1].send_message(getdata_request) - test_nodes[1].wait_for_disconnect() + p2p_conns[1].send_message(getdata_request) + p2p_conns[1].wait_for_disconnect() assert_equal(len(self.nodes[0].getpeerinfo()), 1) self.log.info("Peer 1 disconnected after trying to download old block") @@ -132,39 +130,37 @@ class MaxUploadTest(BitcoinTestFramework): self.log.info("Advancing system time on node to clear counters...") # If we advance the time by 24 hours, then the counters should reset, - # and test_nodes[2] should be able to retrieve the old block. + # and p2p_conns[2] should be able to retrieve the old block. self.nodes[0].setmocktime(int(time.time())) - test_nodes[2].sync_with_ping() - test_nodes[2].send_message(getdata_request) - test_nodes[2].sync_with_ping() - assert_equal(test_nodes[2].block_receive_map[big_old_block], 1) + p2p_conns[2].sync_with_ping() + p2p_conns[2].send_message(getdata_request) + p2p_conns[2].sync_with_ping() + assert_equal(p2p_conns[2].block_receive_map[big_old_block], 1) self.log.info("Peer 2 able to download old block") - [c.disconnect_node() for c in connections] + self.nodes[0].disconnect_p2ps() #stop and start node 0 with 1MB maxuploadtarget, whitelist 127.0.0.1 self.log.info("Restarting nodes with -whitelist=127.0.0.1") self.stop_node(0) self.start_node(0, ["-whitelist=127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000"]) - #recreate/reconnect a test node - test_nodes = [TestNode()] - connections = [NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[0])] - test_nodes[0].add_connection(connections[0]) + # Reconnect to self.nodes[0] + self.nodes[0].add_p2p_connection(TestNode()) - NetworkThread().start() # Start up network handling in another thread - test_nodes[0].wait_for_verack() + network_thread_start() + self.nodes[0].p2p.wait_for_verack() #retrieve 20 blocks which should be enough to break the 1MB limit getdata_request.inv = [CInv(2, big_new_block)] for i in range(20): - test_nodes[0].send_message(getdata_request) - test_nodes[0].sync_with_ping() - assert_equal(test_nodes[0].block_receive_map[big_new_block], i+1) + self.nodes[0].p2p.send_message(getdata_request) + self.nodes[0].p2p.sync_with_ping() + assert_equal(self.nodes[0].p2p.block_receive_map[big_new_block], i+1) getdata_request.inv = [CInv(2, big_old_block)] - test_nodes[0].send_and_ping(getdata_request) + self.nodes[0].p2p.send_and_ping(getdata_request) assert_equal(len(self.nodes[0].getpeerinfo()), 1) #node is still connected because of the whitelist self.log.info("Peer still connected after trying to download old block (whitelisted)") diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index e24dc5a464..e7ce3820d2 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool limiting together/eviction with the wallet.""" @@ -17,10 +17,14 @@ class MempoolLimitTest(BitcoinTestFramework): txouts = gen_return_txouts() relayfee = self.nodes[0].getnetworkinfo()['relayfee'] + self.log.info('Check that mempoolminfee is minrelytxfee') + assert_equal(self.nodes[0].getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) + assert_equal(self.nodes[0].getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + txids = [] utxos = create_confirmed_utxos(relayfee, self.nodes[0], 91) - #create a mempool tx that will be evicted + self.log.info('Create a mempool tx that will be evicted') us0 = utxos.pop() inputs = [{ "txid" : us0["txid"], "vout" : us0["vout"]}] outputs = {self.nodes[0].getnewaddress() : 0.0001} @@ -37,10 +41,14 @@ class MempoolLimitTest(BitcoinTestFramework): txids.append([]) txids[i] = create_lots_of_big_transactions(self.nodes[0], txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) - # by now, the tx should be evicted, check confirmation state + self.log.info('The tx should be evicted by now') assert(txid not in self.nodes[0].getrawmempool()) txdata = self.nodes[0].gettransaction(txid) assert(txdata['confirmations'] == 0) #confirmation should still be 0 + self.log.info('Check that mempoolminfee is larger than minrelytxfee') + assert_equal(self.nodes[0].getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) + assert_greater_than(self.nodes[0].getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + if __name__ == '__main__': MempoolLimitTest().main() diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index b845c75681..a3e872a8c6 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descendant package tracking code.""" diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 92f66be2ff..31a96ec60e 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -57,21 +57,27 @@ class MempoolPersistTest(BitcoinTestFramework): self.log.debug("Send 5 transactions from node2 (to its own address)") for i in range(5): self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) + node2_balance = self.nodes[2].getbalance() self.sync_all() self.log.debug("Verify that node0 and node1 have 5 transactions in their mempools") assert_equal(len(self.nodes[0].getrawmempool()), 5) assert_equal(len(self.nodes[1].getrawmempool()), 5) - self.log.debug("Stop-start node0 and node1. Verify that node0 has the transactions in its mempool and node1 does not.") + self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.") self.stop_nodes() self.start_node(0) self.start_node(1) + self.start_node(2) # Give bitcoind a second to reload the mempool time.sleep(1) wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5) + wait_until(lambda: len(self.nodes[2].getrawmempool()) == 5) assert_equal(len(self.nodes[1].getrawmempool()), 0) + # Verify accounting of mempool transactions after restart is correct + assert_equal(node2_balance, self.nodes[2].getbalance()) + self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() self.start_node(0, extra_args=["-persistmempool=0"]) diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 2803371f5b..d6bb292a58 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool re-org scenarios. diff --git a/test/functional/mempool_resurrect_test.py b/test/functional/mempool_resurrect_test.py index 1263c9306b..83e84da4bc 100755 --- a/test/functional/mempool_resurrect_test.py +++ b/test/functional/mempool_resurrect_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test resurrection of mined transactions when the blockchain is re-organized.""" diff --git a/test/functional/mempool_spendcoinbase.py b/test/functional/mempool_spendcoinbase.py index 6e8a635a76..db0738c08a 100755 --- a/test/functional/mempool_spendcoinbase.py +++ b/test/functional/mempool_spendcoinbase.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test spending coinbase transactions. diff --git a/test/functional/merkle_blocks.py b/test/functional/merkle_blocks.py index b3989a4c54..50e0371fdf 100755 --- a/test/functional/merkle_blocks.py +++ b/test/functional/merkle_blocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test gettxoutproof and verifytxoutproof RPCs.""" diff --git a/test/functional/minchainwork.py b/test/functional/minchainwork.py index c7579d2548..90a3de0e0d 100755 --- a/test/functional/minchainwork.py +++ b/test/functional/minchainwork.py @@ -18,7 +18,7 @@ only succeeds past a given node once its nMinimumChainWork has been exceeded. import time from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import sync_blocks, connect_nodes, assert_equal +from test_framework.util import connect_nodes, assert_equal # 2 hashes required per regtest block (with no difficulty adjustment) REGTEST_WORK_PER_BLOCK = 2 @@ -27,6 +27,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [[], ["-minimumchainwork=0x65"], ["-minimumchainwork=0x65"]] self.node_min_work = [0, 101, 101] @@ -74,6 +75,13 @@ class MinimumChainWorkTest(BitcoinTestFramework): self.nodes[0].generate(1) self.log.info("Verifying nodes are all synced") + + # Because nodes in regtest are all manual connections (eg using + # addnode), node1 should not have disconnected node0. If not for that, + # we'd expect node1 to have disconnected node0 for serving an + # insufficient work chain, in which case we'd need to reconnect them to + # continue the test. + self.sync_all() self.log.info("Blockcounts: %s", [n.getblockcount() for n in self.nodes]) diff --git a/test/functional/mining.py b/test/functional/mining.py index 9aee06864e..569bf71933 100755 --- a/test/functional/mining.py +++ b/test/functional/mining.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mining RPCs diff --git a/test/functional/multi_rpc.py b/test/functional/multi_rpc.py index a2b346f274..01f68344ae 100755 --- a/test/functional/multi_rpc.py +++ b/test/functional/multi_rpc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiple RPC users.""" diff --git a/test/functional/multiwallet.py b/test/functional/multiwallet.py index 6adcc1fd88..12d9e9f48d 100755 --- a/test/functional/multiwallet.py +++ b/test/functional/multiwallet.py @@ -7,6 +7,7 @@ Verify that a bitcoind node can load multiple wallet files """ import os +import shutil from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error @@ -15,10 +16,17 @@ class MultiWalletTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [['-wallet=w1', '-wallet=w2', '-wallet=w3']] + self.extra_args = [['-wallet=w1', '-wallet=w2', '-wallet=w3', '-wallet=w']] + self.supports_cli = True def run_test(self): - assert_equal(set(self.nodes[0].listwallets()), {"w1", "w2", "w3"}) + node = self.nodes[0] + + data_dir = lambda *p: os.path.join(node.datadir, 'regtest', *p) + wallet_dir = lambda *p: data_dir('wallets', *p) + wallet = lambda name: node.get_wallet_rpc(name) + + assert_equal(set(node.listwallets()), {"w1", "w2", "w3", "w"}) self.stop_node(0) @@ -26,19 +34,50 @@ class MultiWalletTest(BitcoinTestFramework): self.assert_start_raises_init_error(0, ['-wallet=w1', '-wallet=w1'], 'Error loading wallet w1. Duplicate -wallet filename specified.') # should not initialize if wallet file is a directory - os.mkdir(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w11')) + os.mkdir(wallet_dir('w11')) self.assert_start_raises_init_error(0, ['-wallet=w11'], 'Error loading wallet w11. -wallet filename must be a regular file.') + # should not initialize if one wallet is a copy of another + shutil.copyfile(wallet_dir('w2'), wallet_dir('w22')) + self.assert_start_raises_init_error(0, ['-wallet=w2', '-wallet=w22'], 'duplicates fileid') + # should not initialize if wallet file is a symlink - os.symlink(os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w1'), os.path.join(self.options.tmpdir, 'node0', 'regtest', 'w12')) + os.symlink(wallet_dir('w1'), wallet_dir('w12')) self.assert_start_raises_init_error(0, ['-wallet=w12'], 'Error loading wallet w12. -wallet filename must be a regular file.') + # should not initialize if the specified walletdir does not exist + self.assert_start_raises_init_error(0, ['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist') + # should not initialize if the specified walletdir is not a directory + not_a_dir = wallet_dir('notadir') + open(not_a_dir, 'a').close() + self.assert_start_raises_init_error(0, ['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') + + # if wallets/ doesn't exist, datadir should be the default wallet dir + wallet_dir2 = data_dir('walletdir') + os.rename(wallet_dir(), wallet_dir2) + self.start_node(0, ['-wallet=w4', '-wallet=w5']) + assert_equal(set(node.listwallets()), {"w4", "w5"}) + w5 = wallet("w5") + w5.generate(1) + self.stop_node(0) + + # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded + os.rename(wallet_dir2, wallet_dir()) + self.start_node(0, ['-wallet=w4', '-wallet=w5', '-walletdir=' + data_dir()]) + assert_equal(set(node.listwallets()), {"w4", "w5"}) + w5 = wallet("w5") + w5_info = w5.getwalletinfo() + assert_equal(w5_info['immature_balance'], 50) + + self.stop_node(0) + self.start_node(0, self.extra_args[0]) - w1 = self.nodes[0].get_wallet_rpc("w1") - w2 = self.nodes[0].get_wallet_rpc("w2") - w3 = self.nodes[0].get_wallet_rpc("w3") - wallet_bad = self.nodes[0].get_wallet_rpc("bad") + w1 = wallet("w1") + w2 = wallet("w2") + w3 = wallet("w3") + w4 = wallet("w") + wallet_bad = wallet("bad") w1.generate(1) @@ -46,7 +85,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", wallet_bad.getwalletinfo) # accessing wallet RPC without using wallet endpoint fails - assert_raises_rpc_error(-19, "Wallet file not specified", self.nodes[0].getwalletinfo) + assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) # check w1 wallet balance w1_info = w1.getwalletinfo() @@ -63,18 +102,26 @@ class MultiWalletTest(BitcoinTestFramework): w3_name = w3.getwalletinfo()['walletname'] assert_equal(w3_name, "w3") - assert_equal({"w1", "w2", "w3"}, {w1_name, w2_name, w3_name}) + w4_name = w4.getwalletinfo()['walletname'] + assert_equal(w4_name, "w") w1.generate(101) assert_equal(w1.getbalance(), 100) assert_equal(w2.getbalance(), 0) assert_equal(w3.getbalance(), 0) + assert_equal(w4.getbalance(), 0) w1.sendtoaddress(w2.getnewaddress(), 1) w1.sendtoaddress(w3.getnewaddress(), 2) + w1.sendtoaddress(w4.getnewaddress(), 3) w1.generate(1) assert_equal(w2.getbalance(), 1) assert_equal(w3.getbalance(), 2) + assert_equal(w4.getbalance(), 3) + + batch = w1.batch([w1.getblockchaininfo.get_request(), w1.getwalletinfo.get_request()]) + assert_equal(batch[0]["result"]["chain"], "regtest") + assert_equal(batch[1]["result"]["walletname"], "w1") if __name__ == '__main__': MultiWalletTest().main() diff --git a/test/functional/node_network_limited.py b/test/functional/node_network_limited.py new file mode 100755 index 0000000000..70415e0168 --- /dev/null +++ b/test/functional/node_network_limited.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Tests NODE_NETWORK_LIMITED. + +Tests that a node configured with -prune=550 signals NODE_NETWORK_LIMITED correctly +and that it responds to getdata requests for blocks correctly: + - send a block within 288 + 2 of the tip + - disconnect peers who request blocks older than that.""" +from test_framework.messages import CInv, msg_getdata +from test_framework.mininode import NODE_BLOOM, NODE_NETWORK_LIMITED, NODE_WITNESS, NetworkThread, P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class P2PIgnoreInv(P2PInterface): + def on_inv(self, message): + # The node will send us invs for other blocks. Ignore them. + pass + + def send_getdata_for_block(self, blockhash): + getdata_request = msg_getdata() + getdata_request.inv.append(CInv(2, int(blockhash, 16))) + self.send_message(getdata_request) + +class NodeNetworkLimitedTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [['-prune=550']] + + def run_test(self): + node = self.nodes[0].add_p2p_connection(P2PIgnoreInv()) + NetworkThread().start() + node.wait_for_verack() + + expected_services = NODE_BLOOM | NODE_WITNESS | NODE_NETWORK_LIMITED + + self.log.info("Check that node has signalled expected services.") + assert_equal(node.nServices, expected_services) + + self.log.info("Check that the localservices is as expected.") + assert_equal(int(self.nodes[0].getnetworkinfo()['localservices'], 16), expected_services) + + self.log.info("Mine enough blocks to reach the NODE_NETWORK_LIMITED range.") + blocks = self.nodes[0].generate(292) + + self.log.info("Make sure we can max retrive block at tip-288.") + node.send_getdata_for_block(blocks[1]) # last block in valid range + node.wait_for_block(int(blocks[1], 16), timeout=3) + + self.log.info("Requesting block at height 2 (tip-289) must fail (ignored).") + node.send_getdata_for_block(blocks[0]) # first block outside of the 288+2 limit + node.wait_for_disconnect(5) + +if __name__ == '__main__': + NodeNetworkLimitedTest().main() diff --git a/test/functional/notifications.py b/test/functional/notifications.py new file mode 100755 index 0000000000..980bef5fc8 --- /dev/null +++ b/test/functional/notifications.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the -alertnotify, -blocknotify and -walletnotify options.""" +import os + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, wait_until, connect_nodes_bi + +class NotificationsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 2 + self.setup_clean_chain = True + + def setup_network(self): + self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") + self.block_filename = os.path.join(self.options.tmpdir, "blocks.txt") + self.tx_filename = os.path.join(self.options.tmpdir, "transactions.txt") + + # -alertnotify and -blocknotify on node0, walletnotify on node1 + self.extra_args = [["-blockversion=2", + "-alertnotify=echo %%s >> %s" % self.alert_filename, + "-blocknotify=echo %%s >> %s" % self.block_filename], + ["-blockversion=211", + "-rescan", + "-walletnotify=echo %%s >> %s" % self.tx_filename]] + super().setup_network() + + def run_test(self): + self.log.info("test -blocknotify") + block_count = 10 + blocks = self.nodes[1].generate(block_count) + + # wait at most 10 seconds for expected file size before reading the content + wait_until(lambda: os.path.isfile(self.block_filename) and os.stat(self.block_filename).st_size >= (block_count * 65), timeout=10) + + # file content should equal the generated blocks hashes + with open(self.block_filename, 'r') as f: + assert_equal(sorted(blocks), sorted(f.read().splitlines())) + + self.log.info("test -walletnotify") + # wait at most 10 seconds for expected file size before reading the content + wait_until(lambda: os.path.isfile(self.tx_filename) and os.stat(self.tx_filename).st_size >= (block_count * 65), timeout=10) + + # file content should equal the generated transaction hashes + txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) + with open(self.tx_filename, 'r') as f: + assert_equal(sorted(txids_rpc), sorted(f.read().splitlines())) + os.remove(self.tx_filename) + + self.log.info("test -walletnotify after rescan") + # restart node to rescan to force wallet notifications + self.restart_node(1) + connect_nodes_bi(self.nodes, 0, 1) + + wait_until(lambda: os.path.isfile(self.tx_filename) and os.stat(self.tx_filename).st_size >= (block_count * 65), timeout=10) + + # file content should equal the generated transaction hashes + txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) + with open(self.tx_filename, 'r') as f: + assert_equal(sorted(txids_rpc), sorted(f.read().splitlines())) + + # Mine another 41 up-version blocks. -alertnotify should trigger on the 51st. + self.log.info("test -alertnotify") + self.nodes[1].generate(41) + self.sync_all() + + # Give bitcoind 10 seconds to write the alert notification + wait_until(lambda: os.path.isfile(self.alert_filename) and os.path.getsize(self.alert_filename), timeout=10) + + with open(self.alert_filename, 'r', encoding='utf8') as f: + alert_text = f.read() + + # Mine more up-version blocks, should not get more alerts: + self.nodes[1].generate(2) + self.sync_all() + + with open(self.alert_filename, 'r', encoding='utf8') as f: + alert_text2 = f.read() + + self.log.info("-alertnotify should not continue notifying for more unknown version blocks") + assert_equal(alert_text, alert_text2) + +if __name__ == '__main__': + NotificationsTest().main() diff --git a/test/functional/nulldummy.py b/test/functional/nulldummy.py index 91c4550653..110b7a655e 100755 --- a/test/functional/nulldummy.py +++ b/test/functional/nulldummy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test NULLDUMMY softfork. @@ -15,7 +15,7 @@ Generate 427 more blocks. from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -from test_framework.mininode import CTransaction, NetworkThread +from test_framework.mininode import CTransaction, network_thread_start from test_framework.blocktools import create_coinbase, create_block, add_witness_commitment from test_framework.script import CScript from io import BytesIO @@ -40,7 +40,9 @@ class NULLDUMMYTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True - self.extra_args = [['-whitelist=127.0.0.1', '-walletprematurewitness']] + # This script tests NULLDUMMY activation, which is part of the 'segwit' deployment, so we go through + # normal segwit activation here (and don't use the default always-on behaviour). + self.extra_args = [['-whitelist=127.0.0.1', '-walletprematurewitness', '-vbparams=segwit:0:999999999999', '-addresstype=legacy']] def run_test(self): self.address = self.nodes[0].getnewaddress() @@ -48,7 +50,7 @@ class NULLDUMMYTest(BitcoinTestFramework): self.wit_address = self.nodes[0].addwitnessaddress(self.address) self.wit_ms_address = self.nodes[0].addwitnessaddress(self.ms_address) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() self.coinbase_blocks = self.nodes[0].generate(2) # Block 2 coinbase_txid = [] for i in self.coinbase_blocks: diff --git a/test/functional/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py index 5b6429b410..672626f15b 100755 --- a/test/functional/p2p-acceptblock.py +++ b/test/functional/p2p-acceptblock.py @@ -1,40 +1,35 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. -Since behavior differs when receiving unrequested blocks from whitelisted peers -versus non-whitelisted peers, this tests the behavior of both (effectively two -separate tests running in parallel). +Setup: two nodes, node0+node1, not connected to each other. Node1 will have +nMinimumChainWork set to 0x10, so it won't process low-work unrequested blocks. -Setup: two nodes, node0 and node1, not connected to each other. Node0 does not -whitelist localhost, but node1 does. They will each be on their own chain for -this test. - -We have one NodeConn connection to each, test_node and white_node respectively. +We have one P2PInterface connection to node0 called test_node, and one to node1 +called min_work_node. The test: 1. Generate one block on each node, to leave IBD. 2. Mine a new block on each tip, and deliver to each node from node's peer. - The tip should advance. + The tip should advance for node0, but node1 should skip processing due to + nMinimumChainWork. + +Node1 is unused in tests 3-7: -3. Mine a block that forks the previous block, and deliver to each node from - corresponding peer. - Node0 should not process this block (just accept the header), because it is - unrequested and doesn't have more work than the tip. - Node1 should process because this is coming from a whitelisted peer. +3. Mine a block that forks from the genesis block, and deliver to test_node. + Node0 should not process this block (just accept the header), because it + is unrequested and doesn't have more or equal work to the tip. -4. Send another block that builds on the forking block. - Node0 should process this block but be stuck on the shorter chain, because - it's missing an intermediate block. - Node1 should reorg to this longer chain. +4a,b. Send another two blocks that build on the forking block. + Node0 should process the second block but be stuck on the shorter chain, + because it's missing an intermediate block. -4b.Send 288 more blocks on the longer chain. +4c.Send 288 more blocks on the longer chain (the number of blocks ahead + we currently store). Node0 should process all but the last block (too far ahead in height). - Send all headers to Node1, and then send the last block in that chain. - Node1 should accept the block because it's coming from a whitelisted peer. 5. Send a duplicate of the block in #3 to Node0. Node0 should not process the block because it is unrequested, and stay on @@ -46,13 +41,21 @@ The test: 7. Send Node0 the missing block again. Node0 should process and the tip should advance. + +8. Create a fork which is invalid at a height longer than the current chain + (ie to which the node will try to reorg) but which has headers built on top + of the invalid block. Check that we get disconnected if we send more headers + on the chain the node now knows to be invalid. + +9. Test Node1 is able to sync when connected to node0 (which should have sufficient + work on its chain). """ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * import time -from test_framework.blocktools import create_block, create_coinbase +from test_framework.blocktools import create_block, create_coinbase, create_transaction class AcceptBlockTest(BitcoinTestFramework): def add_options(self, parser): @@ -63,37 +66,35 @@ class AcceptBlockTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - self.extra_args = [[], ["-whitelist=127.0.0.1"]] + self.extra_args = [[], ["-minimumchainwork=0x10"]] def setup_network(self): # Node0 will be used to test behavior of processing unrequested blocks # from peers which are not whitelisted, while Node1 will be used for # the whitelisted case. + # Node2 will be used for non-whitelisted peers to test the interaction + # with nMinimumChainWork. self.setup_nodes() def run_test(self): # Setup the p2p connections and start up the network thread. - test_node = NodeConnCB() # connects to node0 (not whitelisted) - white_node = NodeConnCB() # connects to node1 (whitelisted) - - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node)) - connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], white_node)) - test_node.add_connection(connections[0]) - white_node.add_connection(connections[1]) + # test_node connects to node0 (not whitelisted) + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + # min_work_node connects to node1 (whitelisted) + min_work_node = self.nodes[1].add_p2p_connection(P2PInterface()) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() # Test logic begins here test_node.wait_for_verack() - white_node.wait_for_verack() + min_work_node.wait_for_verack() - # 1. Have both nodes mine a block (leave IBD) + # 1. Have nodes mine a block (leave IBD) [ n.generate(1) for n in self.nodes ] tips = [ int("0x" + n.getbestblockhash(), 0) for n in self.nodes ] # 2. Send one block that builds on each tip. - # This should be accepted. + # This should be accepted by node0 blocks_h2 = [] # the height 2 blocks on each node's chain block_time = int(time.time()) + 1 for i in range(2): @@ -101,95 +102,120 @@ class AcceptBlockTest(BitcoinTestFramework): blocks_h2[i].solve() block_time += 1 test_node.send_message(msg_block(blocks_h2[0])) - white_node.send_message(msg_block(blocks_h2[1])) + min_work_node.send_message(msg_block(blocks_h2[1])) - [ x.sync_with_ping() for x in [test_node, white_node] ] + for x in [test_node, min_work_node]: + x.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) - assert_equal(self.nodes[1].getblockcount(), 2) - self.log.info("First height 2 block accepted by both nodes") + assert_equal(self.nodes[1].getblockcount(), 1) + self.log.info("First height 2 block accepted by node0; correctly rejected by node1") - # 3. Send another block that builds on the original tip. - blocks_h2f = [] # Blocks at height 2 that fork off the main chain - for i in range(2): - blocks_h2f.append(create_block(tips[i], create_coinbase(2), blocks_h2[i].nTime+1)) - blocks_h2f[i].solve() - test_node.send_message(msg_block(blocks_h2f[0])) - white_node.send_message(msg_block(blocks_h2f[1])) + # 3. Send another block that builds on genesis. + block_h1f = create_block(int("0x" + self.nodes[0].getblockhash(0), 0), create_coinbase(1), block_time) + block_time += 1 + block_h1f.solve() + test_node.send_message(msg_block(block_h1f)) - [ x.sync_with_ping() for x in [test_node, white_node] ] + test_node.sync_with_ping() + tip_entry_found = False for x in self.nodes[0].getchaintips(): - if x['hash'] == blocks_h2f[0].hash: + if x['hash'] == block_h1f.hash: assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) - for x in self.nodes[1].getchaintips(): - if x['hash'] == blocks_h2f[1].hash: - assert_equal(x['status'], "valid-headers") + # 4. Send another two block that build on the fork. + block_h2f = create_block(block_h1f.sha256, create_coinbase(2), block_time) + block_time += 1 + block_h2f.solve() + test_node.send_message(msg_block(block_h2f)) - self.log.info("Second height 2 block accepted only from whitelisted peer") + test_node.sync_with_ping() + # Since the earlier block was not processed by node, the new block + # can't be fully validated. + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_h2f.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) - # 4. Now send another block that builds on the forking chain. - blocks_h3 = [] - for i in range(2): - blocks_h3.append(create_block(blocks_h2f[i].sha256, create_coinbase(3), blocks_h2f[i].nTime+1)) - blocks_h3[i].solve() - test_node.send_message(msg_block(blocks_h3[0])) - white_node.send_message(msg_block(blocks_h3[1])) + # But this block should be accepted by node since it has equal work. + self.nodes[0].getblock(block_h2f.hash) + self.log.info("Second height 2 block accepted, but not reorg'ed to") + + # 4b. Now send another block that builds on the forking chain. + block_h3 = create_block(block_h2f.sha256, create_coinbase(3), block_h2f.nTime+1) + block_h3.solve() + test_node.send_message(msg_block(block_h3)) - [ x.sync_with_ping() for x in [test_node, white_node] ] - # Since the earlier block was not processed by node0, the new block + test_node.sync_with_ping() + # Since the earlier block was not processed by node, the new block # can't be fully validated. + tip_entry_found = False for x in self.nodes[0].getchaintips(): - if x['hash'] == blocks_h3[0].hash: + if x['hash'] == block_h3.hash: assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + self.nodes[0].getblock(block_h3.hash) + + # But this block should be accepted by node since it has more work. + self.nodes[0].getblock(block_h3.hash) + self.log.info("Unrequested more-work block accepted") + + # 4c. Now mine 288 more blocks and deliver; all should be processed but + # the last (height-too-high) on node (as long as its not missing any headers) + tip = block_h3 + all_blocks = [] + for i in range(288): + next_block = create_block(tip.sha256, create_coinbase(i + 4), tip.nTime+1) + next_block.solve() + all_blocks.append(next_block) + tip = next_block + + # Now send the block at height 5 and check that it wasn't accepted (missing header) + test_node.send_message(msg_block(all_blocks[1])) + test_node.sync_with_ping() + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblock, all_blocks[1].hash) + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockheader, all_blocks[1].hash) - # But this block should be accepted by node0 since it has more work. - self.nodes[0].getblock(blocks_h3[0].hash) - self.log.info("Unrequested more-work block accepted from non-whitelisted peer") + # The block at height 5 should be accepted if we provide the missing header, though + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(all_blocks[0])) + test_node.send_message(headers_message) + test_node.send_message(msg_block(all_blocks[1])) + test_node.sync_with_ping() + self.nodes[0].getblock(all_blocks[1].hash) - # Node1 should have accepted and reorged. - assert_equal(self.nodes[1].getblockcount(), 3) - self.log.info("Successfully reorged to length 3 chain from whitelisted peer") + # Now send the blocks in all_blocks + for i in range(288): + test_node.send_message(msg_block(all_blocks[i])) + test_node.sync_with_ping() - # 4b. Now mine 288 more blocks and deliver; all should be processed but - # the last (height-too-high) on node0. Node1 should process the tip if - # we give it the headers chain leading to the tip. - tips = blocks_h3 - headers_message = msg_headers() - all_blocks = [] # node0's blocks - for j in range(2): - for i in range(288): - next_block = create_block(tips[j].sha256, create_coinbase(i + 4), tips[j].nTime+1) - next_block.solve() - if j==0: - test_node.send_message(msg_block(next_block)) - all_blocks.append(next_block) - else: - headers_message.headers.append(CBlockHeader(next_block)) - tips[j] = next_block - - time.sleep(2) # Blocks 1-287 should be accepted, block 288 should be ignored because it's too far ahead for x in all_blocks[:-1]: self.nodes[0].getblock(x.hash) assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[-1].hash) - headers_message.headers.pop() # Ensure the last block is unrequested - white_node.send_message(headers_message) # Send headers leading to tip - white_node.send_message(msg_block(tips[1])) # Now deliver the tip - white_node.sync_with_ping() - self.nodes[1].getblock(tips[1].hash) - self.log.info("Unrequested block far ahead of tip accepted from whitelisted peer") - # 5. Test handling of unrequested block on the node that didn't process # Should still not be processed (even though it has a child that has more # work). - test_node.send_message(msg_block(blocks_h2f[0])) - # Here, if the sleep is too short, the test could falsely succeed (if the - # node hasn't processed the block by the time the sleep returns, and then - # the node processes it and incorrectly advances the tip). - # But this would be caught later on, when we verify that an inv triggers - # a getdata request for this block. + # The node should have requested the blocks at some point, so + # disconnect/reconnect first + + self.nodes[0].disconnect_p2ps() + self.nodes[1].disconnect_p2ps() + network_thread_join() + + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + network_thread_start() + test_node.wait_for_verack() + + test_node.send_message(msg_block(block_h1f)) + test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 2) self.log.info("Unrequested block that would complete more-work chain was ignored") @@ -200,24 +226,98 @@ class AcceptBlockTest(BitcoinTestFramework): with mininode_lock: # Clear state so we can check the getdata request test_node.last_message.pop("getdata", None) - test_node.send_message(msg_inv([CInv(2, blocks_h3[0].sha256)])) + test_node.send_message(msg_inv([CInv(2, block_h3.sha256)])) test_node.sync_with_ping() with mininode_lock: getdata = test_node.last_message["getdata"] # Check that the getdata includes the right block - assert_equal(getdata.inv[0].hash, blocks_h2f[0].sha256) + assert_equal(getdata.inv[0].hash, block_h1f.sha256) self.log.info("Inv at tip triggered getdata for unprocessed block") # 7. Send the missing block for the third time (now it is requested) - test_node.send_message(msg_block(blocks_h2f[0])) + test_node.send_message(msg_block(block_h1f)) test_node.sync_with_ping() assert_equal(self.nodes[0].getblockcount(), 290) + self.nodes[0].getblock(all_blocks[286].hash) + assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, all_blocks[287].hash) self.log.info("Successfully reorged to longer chain from non-whitelisted peer") - [ c.disconnect_node() for c in connections ] + # 8. Create a chain which is invalid at a height longer than the + # current chain, but which has more blocks on top of that + block_289f = create_block(all_blocks[284].sha256, create_coinbase(289), all_blocks[284].nTime+1) + block_289f.solve() + block_290f = create_block(block_289f.sha256, create_coinbase(290), block_289f.nTime+1) + block_290f.solve() + block_291 = create_block(block_290f.sha256, create_coinbase(291), block_290f.nTime+1) + # block_291 spends a coinbase below maturity! + block_291.vtx.append(create_transaction(block_290f.vtx[0], 0, b"42", 1)) + block_291.hashMerkleRoot = block_291.calc_merkle_root() + block_291.solve() + block_292 = create_block(block_291.sha256, create_coinbase(292), block_291.nTime+1) + block_292.solve() + + # Now send all the headers on the chain and enough blocks to trigger reorg + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(block_289f)) + headers_message.headers.append(CBlockHeader(block_290f)) + headers_message.headers.append(CBlockHeader(block_291)) + headers_message.headers.append(CBlockHeader(block_292)) + test_node.send_message(headers_message) + + test_node.sync_with_ping() + tip_entry_found = False + for x in self.nodes[0].getchaintips(): + if x['hash'] == block_292.hash: + assert_equal(x['status'], "headers-only") + tip_entry_found = True + assert(tip_entry_found) + assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) + + test_node.send_message(msg_block(block_289f)) + test_node.send_message(msg_block(block_290f)) + + test_node.sync_with_ping() + self.nodes[0].getblock(block_289f.hash) + self.nodes[0].getblock(block_290f.hash) + + test_node.send_message(msg_block(block_291)) + + # At this point we've sent an obviously-bogus block, wait for full processing + # without assuming whether we will be disconnected or not + try: + # Only wait a short while so the test doesn't take forever if we do get + # disconnected + test_node.sync_with_ping(timeout=1) + except AssertionError: + test_node.wait_for_disconnect() + + self.nodes[0].disconnect_p2ps() + test_node = self.nodes[0].add_p2p_connection(P2PInterface()) + + network_thread_start() + test_node.wait_for_verack() + + # We should have failed reorg and switched back to 290 (but have block 291) + assert_equal(self.nodes[0].getblockcount(), 290) + assert_equal(self.nodes[0].getbestblockhash(), all_blocks[286].hash) + assert_equal(self.nodes[0].getblock(block_291.hash)["confirmations"], -1) + + # Now send a new header on the invalid chain, indicating we're forked off, and expect to get disconnected + block_293 = create_block(block_292.sha256, create_coinbase(293), block_292.nTime+1) + block_293.solve() + headers_message = msg_headers() + headers_message.headers.append(CBlockHeader(block_293)) + test_node.send_message(headers_message) + test_node.wait_for_disconnect() + + # 9. Connect node1 to node0 and ensure it is able to sync + connect_nodes(self.nodes[0], 1) + sync_blocks([self.nodes[0], self.nodes[1]]) + self.log.info("Successfully synced nodes 1 and 0") if __name__ == '__main__': AcceptBlockTest().main() diff --git a/test/functional/p2p-compactblocks.py b/test/functional/p2p-compactblocks.py index 94513d3f43..e98ae31a89 100755 --- a/test/functional/p2p-compactblocks.py +++ b/test/functional/p2p-compactblocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test compact blocks (BIP 152). @@ -15,7 +15,7 @@ from test_framework.blocktools import create_block, create_coinbase, add_witness from test_framework.script import CScript, OP_TRUE # TestNode: A peer we use to send messages to bitcoind, and store responses. -class TestNode(NodeConnCB): +class TestNode(P2PInterface): def __init__(self): super().__init__() self.last_sendcmpct = [] @@ -25,21 +25,21 @@ class TestNode(NodeConnCB): # so we can eg wait until a particular block is announced. self.announced_blockhashes = set() - def on_sendcmpct(self, conn, message): + def on_sendcmpct(self, message): self.last_sendcmpct.append(message) - def on_cmpctblock(self, conn, message): + def on_cmpctblock(self, message): self.block_announced = True self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256) - def on_headers(self, conn, message): + def on_headers(self, message): self.block_announced = True for x in self.last_message["headers"].headers: x.calc_sha256() self.announced_blockhashes.add(x.sha256) - def on_inv(self, conn, message): + def on_inv(self, message): for x in self.last_message["inv"].inv: if x.type == 2: self.block_announced = True @@ -60,7 +60,7 @@ class TestNode(NodeConnCB): msg = msg_getheaders() msg.locator.vHave = locator msg.hashstop = hashstop - self.connection.send_message(msg) + self.send_message(msg) def send_header_for_blocks(self, new_blocks): headers_message = msg_headers() @@ -86,14 +86,16 @@ class TestNode(NodeConnCB): This is used when we want to send a message into the node that we expect will get us disconnected, eg an invalid block.""" self.send_message(message) - wait_until(lambda: not self.connected, timeout=timeout, lock=mininode_lock) + wait_until(lambda: self.state != "connected", timeout=timeout, lock=mininode_lock) class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True # Node0 = pre-segwit, node1 = segwit-aware self.num_nodes = 2 - self.extra_args = [["-vbparams=segwit:0:0"], ["-txindex"]] + # This test was written assuming SegWit is activated using BIP9 at height 432 (3x confirmation window). + # TODO: Rewrite this test to support SegWit being always active. + self.extra_args = [["-vbparams=segwit:0:0"], ["-vbparams=segwit:0:999999999999", "-txindex"]] self.utxos = [] def build_block_on_tip(self, node, segwit=False): @@ -786,23 +788,12 @@ class CompactBlocksTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections and start up the network thread. - self.test_node = TestNode() - self.segwit_node = TestNode() - self.old_node = TestNode() # version 1 peer <--> segwit node - - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node)) - connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], - self.segwit_node, services=NODE_NETWORK|NODE_WITNESS)) - connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], - self.old_node, services=NODE_NETWORK)) - self.test_node.add_connection(connections[0]) - self.segwit_node.add_connection(connections[1]) - self.old_node.add_connection(connections[2]) - - NetworkThread().start() # Start up network handling in another thread - - # Test logic begins here + self.test_node = self.nodes[0].add_p2p_connection(TestNode()) + self.segwit_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + self.old_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK) + + network_thread_start() + self.test_node.wait_for_verack() # We will need UTXOs to construct transactions in later tests. diff --git a/test/functional/p2p-feefilter.py b/test/functional/p2p-feefilter.py index 8c92365ced..47d9c55160 100755 --- a/test/functional/p2p-feefilter.py +++ b/test/functional/p2p-feefilter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of feefilter messages.""" @@ -22,12 +22,12 @@ def allInvsMatch(invsExpected, testnode): time.sleep(1) return False -class TestNode(NodeConnCB): +class TestNode(P2PInterface): def __init__(self): super().__init__() self.txinvs = [] - def on_inv(self, conn, message): + def on_inv(self, message): for i in message.inv: if (i.type == 1): self.txinvs.append(hashToHex(i.hash)) @@ -48,25 +48,23 @@ class FeeFilterTest(BitcoinTestFramework): sync_blocks(self.nodes) # Setup the p2p connections and start up the network thread. - test_node = TestNode() - connection = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node) - test_node.add_connection(connection) - NetworkThread().start() - test_node.wait_for_verack() + self.nodes[0].add_p2p_connection(TestNode()) + network_thread_start() + self.nodes[0].p2p.wait_for_verack() # Test that invs are received for all txs at feerate of 20 sat/byte node1.settxfee(Decimal("0.00020000")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, test_node)) - test_node.clear_invs() + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() # Set a filter of 15 sat/byte - test_node.send_and_ping(msg_feefilter(15000)) + self.nodes[0].p2p.send_and_ping(msg_feefilter(15000)) # Test that txs are still being received (paying 20 sat/byte) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, test_node)) - test_node.clear_invs() + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() # Change tx fee rate to 10 sat/byte and test they are no longer received node1.settxfee(Decimal("0.00010000")) @@ -82,14 +80,14 @@ class FeeFilterTest(BitcoinTestFramework): # as well. node0.settxfee(Decimal("0.00020000")) txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] - assert(allInvsMatch(txids, test_node)) - test_node.clear_invs() + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() # Remove fee filter and check that txs are received again - test_node.send_and_ping(msg_feefilter(0)) + self.nodes[0].p2p.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, test_node)) - test_node.clear_invs() + assert(allInvsMatch(txids, self.nodes[0].p2p)) + self.nodes[0].p2p.clear_invs() if __name__ == '__main__': FeeFilterTest().main() diff --git a/test/functional/p2p-fingerprint.py b/test/functional/p2p-fingerprint.py new file mode 100755 index 0000000000..93ef73e25e --- /dev/null +++ b/test/functional/p2p-fingerprint.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test various fingerprinting protections. + +If an stale block more than a month old or its header are requested by a peer, +the node should pretend that it does not have it to avoid fingerprinting. +""" + +import time + +from test_framework.blocktools import (create_block, create_coinbase) +from test_framework.mininode import ( + CInv, + P2PInterface, + msg_headers, + msg_block, + msg_getdata, + msg_getheaders, + network_thread_start, + wait_until, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + +class P2PFingerprintTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + # Build a chain of blocks on top of given one + def build_chain(self, nblocks, prev_hash, prev_height, prev_median_time): + blocks = [] + for _ in range(nblocks): + coinbase = create_coinbase(prev_height + 1) + block_time = prev_median_time + 1 + block = create_block(int(prev_hash, 16), coinbase, block_time) + block.solve() + + blocks.append(block) + prev_hash = block.hash + prev_height += 1 + prev_median_time = block_time + return blocks + + # Send a getdata request for a given block hash + def send_block_request(self, block_hash, node): + msg = msg_getdata() + msg.inv.append(CInv(2, block_hash)) # 2 == "Block" + node.send_message(msg) + + # Send a getheaders request for a given single block hash + def send_header_request(self, block_hash, node): + msg = msg_getheaders() + msg.hashstop = block_hash + node.send_message(msg) + + # Check whether last block received from node has a given hash + def last_block_equals(self, expected_hash, node): + block_msg = node.last_message.get("block") + return block_msg and block_msg.block.rehash() == expected_hash + + # Check whether last block header received from node has a given hash + def last_header_equals(self, expected_hash, node): + headers_msg = node.last_message.get("headers") + return (headers_msg and + headers_msg.headers and + headers_msg.headers[0].rehash() == expected_hash) + + # Checks that stale blocks timestamped more than a month ago are not served + # by the node while recent stale blocks and old active chain blocks are. + # This does not currently test that stale blocks timestamped within the + # last month but that have over a month's worth of work are also withheld. + def run_test(self): + node0 = self.nodes[0].add_p2p_connection(P2PInterface()) + + network_thread_start() + node0.wait_for_verack() + + # Set node time to 60 days ago + self.nodes[0].setmocktime(int(time.time()) - 60 * 24 * 60 * 60) + + # Generating a chain of 10 blocks + block_hashes = self.nodes[0].generate(nblocks=10) + + # Create longer chain starting 2 blocks before current tip + height = len(block_hashes) - 2 + block_hash = block_hashes[height - 1] + block_time = self.nodes[0].getblockheader(block_hash)["mediantime"] + 1 + new_blocks = self.build_chain(5, block_hash, height, block_time) + + # Force reorg to a longer chain + node0.send_message(msg_headers(new_blocks)) + node0.wait_for_getdata() + for block in new_blocks: + node0.send_and_ping(msg_block(block)) + + # Check that reorg succeeded + assert_equal(self.nodes[0].getblockcount(), 13) + + stale_hash = int(block_hashes[-1], 16) + + # Check that getdata request for stale block succeeds + self.send_block_request(stale_hash, node0) + test_function = lambda: self.last_block_equals(stale_hash, node0) + wait_until(test_function, timeout=3) + + # Check that getheader request for stale block header succeeds + self.send_header_request(stale_hash, node0) + test_function = lambda: self.last_header_equals(stale_hash, node0) + wait_until(test_function, timeout=3) + + # Longest chain is extended so stale is much older than chain tip + self.nodes[0].setmocktime(0) + tip = self.nodes[0].generate(nblocks=1)[0] + assert_equal(self.nodes[0].getblockcount(), 14) + + # Send getdata & getheaders to refresh last received getheader message + block_hash = int(tip, 16) + self.send_block_request(block_hash, node0) + self.send_header_request(block_hash, node0) + node0.sync_with_ping() + + # Request for very old stale block should now fail + self.send_block_request(stale_hash, node0) + time.sleep(3) + assert not self.last_block_equals(stale_hash, node0) + + # Request for very old stale block header should now fail + self.send_header_request(stale_hash, node0) + time.sleep(3) + assert not self.last_header_equals(stale_hash, node0) + + # Verify we can fetch very old blocks and headers on the active chain + block_hash = int(block_hashes[2], 16) + self.send_block_request(block_hash, node0) + self.send_header_request(block_hash, node0) + node0.sync_with_ping() + + self.send_block_request(block_hash, node0) + test_function = lambda: self.last_block_equals(block_hash, node0) + wait_until(test_function, timeout=3) + + self.send_header_request(block_hash, node0) + test_function = lambda: self.last_header_equals(block_hash, node0) + wait_until(test_function, timeout=3) + +if __name__ == '__main__': + P2PFingerprintTest().main() diff --git a/test/functional/p2p-fullblocktest.py b/test/functional/p2p-fullblocktest.py index 1d969fc7c1..fe9bbda14b 100755 --- a/test/functional/p2p-fullblocktest.py +++ b/test/functional/p2p-fullblocktest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test block processing. @@ -18,9 +18,10 @@ from test_framework.blocktools import * import time from test_framework.key import CECKey from test_framework.script import * +from test_framework.mininode import network_thread_start import struct -class PreviousSpendableOutput(object): +class PreviousSpendableOutput(): def __init__(self, tx = CTransaction(), n = -1): self.tx = tx self.n = n # the output we're spending @@ -35,12 +36,15 @@ class CBrokenBlock(CBlock): self.vtx = copy.deepcopy(base_block.vtx) self.hashMerkleRoot = self.calc_merkle_root() - def serialize(self): + def serialize(self, with_witness=False): r = b"" r += super(CBlock, self).serialize() r += struct.pack("<BQ", 255, len(self.vtx)) for tx in self.vtx: - r += tx.serialize() + if with_witness: + r += tx.serialize_with_witness() + else: + r += tx.serialize_without_witness() return r def normal_serialize(self): @@ -68,7 +72,7 @@ class FullBlockTest(ComparisonTestFramework): def run_test(self): self.test = TestManager(self, self.options.tmpdir) self.test.add_all_connections(self.nodes) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() self.test.run() def add_transactions_to_block(self, block, tx_list): diff --git a/test/functional/p2p-leaktests.py b/test/functional/p2p-leaktests.py index 1dc8f72cd6..ce4e6e9144 100755 --- a/test/functional/p2p-leaktests.py +++ b/test/functional/p2p-leaktests.py @@ -20,7 +20,7 @@ from test_framework.util import * banscore = 10 -class CLazyNode(NodeConnCB): +class CLazyNode(P2PInterface): def __init__(self): super().__init__() self.unexpected_msg = False @@ -30,44 +30,42 @@ class CLazyNode(NodeConnCB): self.unexpected_msg = True self.log.info("should not have received message: %s" % message.command) - def on_open(self, conn): - self.connected = True + def on_open(self): self.ever_connected = True - def on_version(self, conn, message): self.bad_message(message) - def on_verack(self, conn, message): self.bad_message(message) - def on_reject(self, conn, message): self.bad_message(message) - def on_inv(self, conn, message): self.bad_message(message) - def on_addr(self, conn, message): self.bad_message(message) - def on_alert(self, conn, message): self.bad_message(message) - def on_getdata(self, conn, message): self.bad_message(message) - def on_getblocks(self, conn, message): self.bad_message(message) - def on_tx(self, conn, message): self.bad_message(message) - def on_block(self, conn, message): self.bad_message(message) - def on_getaddr(self, conn, message): self.bad_message(message) - def on_headers(self, conn, message): self.bad_message(message) - def on_getheaders(self, conn, message): self.bad_message(message) - def on_ping(self, conn, message): self.bad_message(message) - def on_mempool(self, conn): self.bad_message(message) - def on_pong(self, conn, message): self.bad_message(message) - def on_feefilter(self, conn, message): self.bad_message(message) - def on_sendheaders(self, conn, message): self.bad_message(message) - def on_sendcmpct(self, conn, message): self.bad_message(message) - def on_cmpctblock(self, conn, message): self.bad_message(message) - def on_getblocktxn(self, conn, message): self.bad_message(message) - def on_blocktxn(self, conn, message): self.bad_message(message) + def on_version(self, message): self.bad_message(message) + def on_verack(self, message): self.bad_message(message) + def on_reject(self, message): self.bad_message(message) + def on_inv(self, message): self.bad_message(message) + def on_addr(self, message): self.bad_message(message) + def on_getdata(self, message): self.bad_message(message) + def on_getblocks(self, message): self.bad_message(message) + def on_tx(self, message): self.bad_message(message) + def on_block(self, message): self.bad_message(message) + def on_getaddr(self, message): self.bad_message(message) + def on_headers(self, message): self.bad_message(message) + def on_getheaders(self, message): self.bad_message(message) + def on_ping(self, message): self.bad_message(message) + def on_mempool(self, message): self.bad_message(message) + def on_pong(self, message): self.bad_message(message) + def on_feefilter(self, message): self.bad_message(message) + def on_sendheaders(self, message): self.bad_message(message) + def on_sendcmpct(self, message): self.bad_message(message) + def on_cmpctblock(self, message): self.bad_message(message) + def on_getblocktxn(self, message): self.bad_message(message) + def on_blocktxn(self, message): self.bad_message(message) # Node that never sends a version. We'll use this to send a bunch of messages # anyway, and eventually get disconnected. class CNodeNoVersionBan(CLazyNode): # send a bunch of veracks without sending a message. This should get us disconnected. # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes - def on_open(self, conn): - super().on_open(conn) + def on_open(self): + super().on_open() for i in range(banscore): self.send_message(msg_verack()) - def on_reject(self, conn, message): pass + def on_reject(self, message): pass # Node that never sends a version. This one just sits idle and hopes to receive # any message (it shouldn't!) @@ -81,15 +79,15 @@ class CNodeNoVerackIdle(CLazyNode): self.version_received = False super().__init__() - def on_reject(self, conn, message): pass - def on_verack(self, conn, message): pass + def on_reject(self, message): pass + def on_verack(self, message): pass # When version is received, don't reply with a verack. Instead, see if the # node will give us a message that it shouldn't. This is not an exhaustive # list! - def on_version(self, conn, message): + def on_version(self, message): self.version_received = True - conn.send_message(msg_ping()) - conn.send_message(msg_getaddr()) + self.send_message(msg_ping()) + self.send_message(msg_getaddr()) class P2PLeakTest(BitcoinTestFramework): def set_test_params(self): @@ -97,26 +95,15 @@ class P2PLeakTest(BitcoinTestFramework): self.extra_args = [['-banscore='+str(banscore)]] def run_test(self): - no_version_bannode = CNodeNoVersionBan() - no_version_idlenode = CNodeNoVersionIdle() - no_verack_idlenode = CNodeNoVerackIdle() - unsupported_service_bit5_node = CLazyNode() - unsupported_service_bit7_node = CLazyNode() - self.nodes[0].setmocktime(1501545600) # August 1st 2017 - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_version_bannode, send_version=False)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_version_idlenode, send_version=False)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], no_verack_idlenode)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], unsupported_service_bit5_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], unsupported_service_bit7_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7)) - no_version_bannode.add_connection(connections[0]) - no_version_idlenode.add_connection(connections[1]) - no_verack_idlenode.add_connection(connections[2]) - unsupported_service_bit5_node.add_connection(connections[3]) - unsupported_service_bit7_node.add_connection(connections[4]) - - NetworkThread().start() # Start up network handling in another thread + + no_version_bannode = self.nodes[0].add_p2p_connection(CNodeNoVersionBan(), send_version=False) + no_version_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVersionIdle(), send_version=False) + no_verack_idlenode = self.nodes[0].add_p2p_connection(CNodeNoVerackIdle()) + unsupported_service_bit5_node = self.nodes[0].add_p2p_connection(CLazyNode(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5) + unsupported_service_bit7_node = self.nodes[0].add_p2p_connection(CLazyNode(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7) + + network_thread_start() wait_until(lambda: no_version_bannode.ever_connected, timeout=10, lock=mininode_lock) wait_until(lambda: no_version_idlenode.ever_connected, timeout=10, lock=mininode_lock) @@ -131,16 +118,17 @@ class P2PLeakTest(BitcoinTestFramework): time.sleep(5) #This node should have been banned - assert not no_version_bannode.connected + assert no_version_bannode.state != "connected" # These nodes should have been disconnected - assert not unsupported_service_bit5_node.connected - assert not unsupported_service_bit7_node.connected + assert unsupported_service_bit5_node.state != "connected" + assert unsupported_service_bit7_node.state != "connected" - [conn.disconnect_node() for conn in connections] + self.nodes[0].disconnect_p2ps() - # Wait until all connections are closed + # Wait until all connections are closed and the network thread has terminated wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0) + network_thread_join() # Make sure no unexpected messages came in assert(no_version_bannode.unexpected_msg == False) @@ -152,15 +140,11 @@ class P2PLeakTest(BitcoinTestFramework): self.log.info("Service bits 5 and 7 are allowed after August 1st 2018") self.nodes[0].setmocktime(1533168000) # August 2nd 2018 - allowed_service_bit5_node = NodeConnCB() - allowed_service_bit7_node = NodeConnCB() - - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], allowed_service_bit5_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], allowed_service_bit7_node, services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7)) - allowed_service_bit5_node.add_connection(connections[5]) - allowed_service_bit7_node.add_connection(connections[6]) + allowed_service_bit5_node = self.nodes[0].add_p2p_connection(P2PInterface(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_5) + allowed_service_bit7_node = self.nodes[0].add_p2p_connection(P2PInterface(), services=NODE_NETWORK|NODE_UNSUPPORTED_SERVICE_BIT_7) - NetworkThread().start() # Network thread stopped when all previous NodeConnCBs disconnected. Restart it + # Network thread stopped when all previous P2PInterfaces disconnected. Restart it + network_thread_start() wait_until(lambda: allowed_service_bit5_node.message_count["verack"], lock=mininode_lock) wait_until(lambda: allowed_service_bit7_node.message_count["verack"], lock=mininode_lock) diff --git a/test/functional/p2p-mempool.py b/test/functional/p2p-mempool.py index 40fcde2605..485a8af3d0 100755 --- a/test/functional/p2p-mempool.py +++ b/test/functional/p2p-mempool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test p2p mempool message. @@ -19,16 +19,14 @@ class P2PMempoolTests(BitcoinTestFramework): self.extra_args = [["-peerbloomfilters=0"]] def run_test(self): - #connect a mininode - aTestNode = NodeConnCB() - node = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], aTestNode) - aTestNode.add_connection(node) - NetworkThread().start() - aTestNode.wait_for_verack() + # Add a p2p connection + self.nodes[0].add_p2p_connection(P2PInterface()) + network_thread_start() + self.nodes[0].p2p.wait_for_verack() #request mempool - aTestNode.send_message(msg_mempool()) - aTestNode.wait_for_disconnect() + self.nodes[0].p2p.send_message(msg_mempool()) + self.nodes[0].p2p.wait_for_disconnect() #mininode must be disconnected at this point assert_equal(len(self.nodes[0].getpeerinfo()), 0) diff --git a/test/functional/p2p-segwit.py b/test/functional/p2p-segwit.py index a9ef47559b..20e4805df0 100755 --- a/test/functional/p2p-segwit.py +++ b/test/functional/p2p-segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test segwit transactions and blocks on P2P network.""" @@ -25,18 +25,46 @@ MAX_SIGOP_COST = 80000 # Calculate the virtual size of a witness block: # (base + witness/4) def get_virtual_size(witness_block): - base_size = len(witness_block.serialize()) + base_size = len(witness_block.serialize(with_witness=False)) total_size = len(witness_block.serialize(with_witness=True)) # the "+3" is so we round up vsize = int((3*base_size + total_size + 3)/4) return vsize -class TestNode(NodeConnCB): +def test_transaction_acceptance(rpc, p2p, tx, with_witness, accepted, reason=None): + """Send a transaction to the node and check that it's accepted to the mempool + + - Submit the transaction over the p2p interface + - use the getrawmempool rpc to check for acceptance.""" + tx_message = msg_tx(tx) + if with_witness: + tx_message = msg_witness_tx(tx) + p2p.send_message(tx_message) + p2p.sync_with_ping() + assert_equal(tx.hash in rpc.getrawmempool(), accepted) + if (reason != None and not accepted): + # Check the rejection reason as well. + with mininode_lock: + assert_equal(p2p.last_message["reject"].reason, reason) + +def test_witness_block(rpc, p2p, block, accepted, with_witness=True): + """Send a block to the node and check that it's accepted + + - Submit the block over the p2p interface + - use the getbestblockhash rpc to check for acceptance.""" + if with_witness: + p2p.send_message(msg_witness_block(block)) + else: + p2p.send_message(msg_block(block)) + p2p.sync_with_ping() + assert_equal(rpc.getbestblockhash() == block.hash, accepted) + +class TestNode(P2PInterface): def __init__(self): super().__init__() self.getdataset = set() - def on_getdata(self, conn, message): + def on_getdata(self, message): for inv in message.inv: self.getdataset.add(inv.hash) @@ -67,29 +95,8 @@ class TestNode(NodeConnCB): self.wait_for_block(blockhash, timeout) return self.last_message["block"].block - def test_transaction_acceptance(self, tx, with_witness, accepted, reason=None): - tx_message = msg_tx(tx) - if with_witness: - tx_message = msg_witness_tx(tx) - self.send_message(tx_message) - self.sync_with_ping() - assert_equal(tx.hash in self.connection.rpc.getrawmempool(), accepted) - if (reason != None and not accepted): - # Check the rejection reason as well. - with mininode_lock: - assert_equal(self.last_message["reject"].reason, reason) - - # Test whether a witness block had the correct effect on the tip - def test_witness_block(self, block, accepted, with_witness=True): - if with_witness: - self.send_message(msg_witness_block(block)) - else: - self.send_message(msg_block(block)) - self.sync_with_ping() - assert_equal(self.connection.rpc.getbestblockhash() == block.hash, accepted) - # Used to keep track of anyone-can-spend outputs that we can use in the tests -class UTXO(object): +class UTXO(): def __init__(self, sha256, n, nValue): self.sha256 = sha256 self.n = n @@ -111,7 +118,8 @@ class SegWitTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-whitelist=127.0.0.1"], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0"], ["-whitelist=127.0.0.1", "-vbparams=segwit:0:0"]] + # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. + self.extra_args = [["-whitelist=127.0.0.1", "-vbparams=segwit:0:999999999999"], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0", "-vbparams=segwit:0:999999999999"], ["-whitelist=127.0.0.1", "-vbparams=segwit:0:0"]] def setup_network(self): self.setup_nodes() @@ -140,7 +148,7 @@ class SegWitTest(BitcoinTestFramework): ''' Individual tests ''' def test_witness_services(self): self.log.info("Verifying NODE_WITNESS service bit") - assert((self.test_node.connection.nServices & NODE_WITNESS) != 0) + assert((self.test_node.nServices & NODE_WITNESS) != 0) # See if sending a regular transaction works, and create a utxo @@ -199,7 +207,7 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx]) # Sending witness data before activation is not allowed (anti-spam # rule). - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # TODO: fix synchronization so we can test reject reason # Right now, bitcoind delays sending reject messages for blocks # until the future, making synchronization here difficult. @@ -226,7 +234,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) tx2.vout.append(CTxOut(tx.vout[0].nValue-1000, scriptPubKey)) tx2.rehash() - self.test_node.test_transaction_acceptance(tx2, False, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, False, True) self.nodes[0].generate(1) sync_blocks(self.nodes) @@ -243,18 +251,18 @@ class SegWitTest(BitcoinTestFramework): tx3.rehash() # Note that this should be rejected for the premature witness reason, # rather than a policy check, since segwit hasn't activated yet. - self.std_node.test_transaction_acceptance(tx3, True, False, b'no-witness-yet') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, True, False, b'no-witness-yet') # If we send without witness, it should be accepted. - self.std_node.test_transaction_acceptance(tx3, False, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, False, True) # Now create a new anyone-can-spend utxo for the next test. tx4 = CTransaction() tx4.vin.append(CTxIn(COutPoint(tx3.sha256, 0), CScript([p2sh_program]))) tx4.vout.append(CTxOut(tx3.vout[0].nValue-1000, CScript([OP_TRUE]))) tx4.rehash() - self.test_node.test_transaction_acceptance(tx3, False, True) - self.test_node.test_transaction_acceptance(tx4, False, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, False, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx4, False, True) self.nodes[0].generate(1) sync_blocks(self.nodes) @@ -315,7 +323,7 @@ class SegWitTest(BitcoinTestFramework): assert(msg_witness_block(block).serialize() != msg_block(block).serialize()) # This empty block should be valid. - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Try to tweak the nonce block_2 = self.build_next_block() @@ -326,7 +334,7 @@ class SegWitTest(BitcoinTestFramework): assert(block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1]) # This should also be valid. - self.test_node.test_witness_block(block_2, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_2, accepted=True) # Now test commitments with actual transactions assert (len(self.utxo) > 0) @@ -359,7 +367,7 @@ class SegWitTest(BitcoinTestFramework): block_3.rehash() block_3.solve() - self.test_node.test_witness_block(block_3, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=False) # Add a different commitment with different nonce, but in the # right location, and with some funds burned(!). @@ -373,7 +381,7 @@ class SegWitTest(BitcoinTestFramework): block_3.rehash() assert(len(block_3.vtx[0].vout) == 4) # 3 OP_returns block_3.solve() - self.test_node.test_witness_block(block_3, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=True) # Finally test that a block with no witness transactions can # omit the commitment. @@ -385,7 +393,7 @@ class SegWitTest(BitcoinTestFramework): block_4.vtx.append(tx3) block_4.hashMerkleRoot = block_4.calc_merkle_root() block_4.solve() - self.test_node.test_witness_block(block_4, with_witness=False, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_4, with_witness=False, accepted=True) # Update available utxo's for use in later test. self.utxo.pop(0) @@ -426,11 +434,11 @@ class SegWitTest(BitcoinTestFramework): # Change the nonce -- should not cause the block to be permanently # failed block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ ser_uint256(1) ] - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Changing the witness nonce doesn't change the block hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ ser_uint256(0) ] - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) def test_witness_block_size(self): @@ -495,7 +503,7 @@ class SegWitTest(BitcoinTestFramework): # limit assert(len(block.serialize(True)) > 2*1024*1024) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now resize the second transaction to make the block fit. cur_length = len(block.vtx[-1].wit.vtxinwit[0].scriptWitness.stack[0]) @@ -505,7 +513,7 @@ class SegWitTest(BitcoinTestFramework): block.solve() assert(get_virtual_size(block) == MAX_BLOCK_BASE_SIZE) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Update available utxo's self.utxo.pop(0) @@ -572,7 +580,7 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx]) # Extra witness data should not be allowed. - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Try extra signature data. Ok if we're not spending a witness output. block.vtx[1].wit.vtxinwit = [] @@ -581,7 +589,7 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Now try extra witness/signature data on an input that DOES require a # witness @@ -597,7 +605,7 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx2]) # This has extra witness data, so it should fail. - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now get rid of the extra witness, but add extra scriptSig data tx2.vin[0].scriptSig = CScript([OP_TRUE]) @@ -609,7 +617,7 @@ class SegWitTest(BitcoinTestFramework): block.solve() # This has extra signature data for a witness input, so it should fail. - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now get rid of the extra scriptsig on the witness input, and verify # success (even with extra scriptsig data in the non-witness input) @@ -618,7 +626,7 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Update utxo for later tests self.utxo.pop(0) @@ -651,14 +659,14 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() self.update_witness_block_with_transactions(block, [tx, tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now reduce the length of the stack element tx2.wit.vtxinwit[0].scriptWitness.stack[0] = b'a'*(MAX_SCRIPT_ELEMENT_SIZE) add_witness_commitment(block) block.solve() - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Update the utxo for later tests self.utxo.pop() @@ -693,7 +701,7 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx, tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Try again with one less byte in the witness program witness_program = CScript([b'a'*520]*19 + [OP_DROP]*62 + [OP_TRUE]) @@ -708,7 +716,7 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx, tx2]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) @@ -734,7 +742,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Try various ways to spend tx that should all break. # This "broken" transaction serializer will not normalize @@ -769,7 +777,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now try using a too short vtxinwit tx2.wit.vtxinwit.pop() @@ -777,7 +785,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now make one of the intermediate witnesses be incorrect tx2.wit.vtxinwit.append(CTxInWitness()) @@ -786,13 +794,13 @@ class SegWitTest(BitcoinTestFramework): block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Fix the broken witness and the block should be accepted. tx2.wit.vtxinwit[5].scriptWitness.stack = [b'a', witness_program] block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) @@ -832,11 +840,11 @@ class SegWitTest(BitcoinTestFramework): # its from) assert_equal(len(self.nodes[0].getrawmempool()), 0) assert_equal(len(self.nodes[1].getrawmempool()), 0) - self.old_node.test_transaction_acceptance(tx, with_witness=True, accepted=False) - self.test_node.test_transaction_acceptance(tx, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.old_node, tx, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=True, accepted=False) # But eliminating the witness should fix it - self.test_node.test_transaction_acceptance(tx, with_witness=False, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) # Cleanup: mine the first transaction and update utxo self.nodes[0].generate(1) @@ -868,11 +876,11 @@ class SegWitTest(BitcoinTestFramework): # Verify that unnecessary witnesses are rejected. self.test_node.announce_tx_and_wait_for_getdata(tx) assert_equal(len(self.nodes[0].getrawmempool()), 0) - self.test_node.test_transaction_acceptance(tx, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=True, accepted=False) # Verify that removing the witness succeeds. self.test_node.announce_tx_and_wait_for_getdata(tx) - self.test_node.test_transaction_acceptance(tx, with_witness=False, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) # Now try to add extra witness data to a valid witness tx. witness_program = CScript([OP_TRUE]) @@ -897,24 +905,24 @@ class SegWitTest(BitcoinTestFramework): # Node will not be blinded to the transaction self.std_node.announce_tx_and_wait_for_getdata(tx3) - self.std_node.test_transaction_acceptance(tx3, True, False, b'tx-size') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, True, False, b'tx-size') self.std_node.announce_tx_and_wait_for_getdata(tx3) - self.std_node.test_transaction_acceptance(tx3, True, False, b'tx-size') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx3, True, False, b'tx-size') # Remove witness stuffing, instead add extra witness push on stack tx3.vout[0] = CTxOut(tx2.vout[0].nValue-1000, CScript([OP_TRUE])) tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)]), witness_program ] tx3.rehash() - self.test_node.test_transaction_acceptance(tx2, with_witness=True, accepted=True) - self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=False) # Get rid of the extra witness, and verify acceptance. tx3.wit.vtxinwit[0].scriptWitness.stack = [ witness_program ] # Also check that old_node gets a tx announcement, even though this is # a witness transaction. self.old_node.wait_for_inv([CInv(1, tx2.sha256)]) # wait until tx2 was inv'ed - self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=True) self.old_node.wait_for_inv([CInv(1, tx3.sha256)]) # Test that getrawtransaction returns correct witness information @@ -953,20 +961,20 @@ class SegWitTest(BitcoinTestFramework): self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) - self.test_node.test_witness_block(block1, True) + test_witness_block(self.nodes[0].rpc, self.test_node, block1, True) block2 = self.build_next_block(nVersion=4) block2.solve() self.test_node.announce_block_and_wait_for_getdata(block2, use_header=True) assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) - self.test_node.test_witness_block(block2, True) + test_witness_block(self.nodes[0].rpc, self.test_node, block2, True) block3 = self.build_next_block(nVersion=(VB_TOP_BITS | (1<<15))) block3.solve() self.test_node.announce_block_and_wait_for_getdata(block3, use_header=True) assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) - self.test_node.test_witness_block(block3, True) + test_witness_block(self.nodes[0].rpc, self.test_node, block3, True) # Check that we can getdata for witness blocks or regular blocks, # and the right thing happens. @@ -996,7 +1004,7 @@ class SegWitTest(BitcoinTestFramework): # This gives us a witness commitment. assert(len(block.vtx[0].wit.vtxinwit) == 1) assert(len(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack) == 1) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Now try to retrieve it... rpc_block = self.nodes[0].getblock(block.hash, False) non_wit_block = self.test_node.request_block(block.sha256, 2) @@ -1050,7 +1058,7 @@ class SegWitTest(BitcoinTestFramework): p2sh_tx.rehash() # Mine it on test_node to create the confirmed output. - self.test_node.test_transaction_acceptance(p2sh_tx, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_tx, with_witness=True, accepted=True) self.nodes[0].generate(1) sync_blocks(self.nodes) @@ -1062,7 +1070,7 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(8000, scriptPubKey)) # Might burn this later tx.rehash() - self.std_node.test_transaction_acceptance(tx, with_witness=True, accepted=segwit_activated) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx, with_witness=True, accepted=segwit_activated) # Now create something that looks like a P2PKH output. This won't be spendable. scriptPubKey = CScript([OP_0, hash160(witness_hash)]) @@ -1079,7 +1087,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vout = [CTxOut(p2sh_tx.vout[0].nValue-1000, scriptPubKey)] tx2.rehash() - self.std_node.test_transaction_acceptance(tx2, with_witness=True, accepted=segwit_activated) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx2, with_witness=True, accepted=segwit_activated) # Now update self.utxo for later tests. tx3 = CTransaction() @@ -1092,13 +1100,13 @@ class SegWitTest(BitcoinTestFramework): tx3.wit.vtxinwit.append(CTxInWitness()) tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program] tx3.rehash() - self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=True) else: # tx and tx2 didn't go anywhere; just clean up the p2sh_tx output. tx3.vin = [CTxIn(COutPoint(p2sh_tx.sha256, 0), CScript([witness_program]))] tx3.vout = [CTxOut(p2sh_tx.vout[0].nValue-1000, witness_program)] tx3.rehash() - self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=True) self.nodes[0].generate(1) sync_blocks(self.nodes) @@ -1122,7 +1130,7 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) self.utxo.pop(0) for i in range(NUM_TESTS): self.utxo.append(UTXO(tx.sha256, i, split_value)) @@ -1141,8 +1149,8 @@ class SegWitTest(BitcoinTestFramework): tx.vin = [CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")] tx.vout = [CTxOut(self.utxo[0].nValue-1000, scriptPubKey)] tx.rehash() - self.std_node.test_transaction_acceptance(tx, with_witness=True, accepted=False) - self.test_node.test_transaction_acceptance(tx, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=True, accepted=True) self.utxo.pop(0) temp_utxo.append(UTXO(tx.sha256, 0, tx.vout[0].nValue)) @@ -1161,8 +1169,8 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() # Gets accepted to test_node, because standardness of outputs isn't # checked with fRequireStandard - self.test_node.test_transaction_acceptance(tx2, with_witness=True, accepted=True) - self.std_node.test_transaction_acceptance(tx2, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, tx2, with_witness=True, accepted=False) temp_utxo.pop() # last entry in temp_utxo was the output we just spent temp_utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) @@ -1178,7 +1186,7 @@ class SegWitTest(BitcoinTestFramework): tx3.rehash() # Spending a higher version witness output is not allowed by policy, # even with fRequireStandard=false. - self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, with_witness=True, accepted=False) self.test_node.sync_with_ping() with mininode_lock: assert(b"reserved for soft-fork upgrades" in self.test_node.last_message["reject"].reason) @@ -1186,7 +1194,7 @@ class SegWitTest(BitcoinTestFramework): # Building a block with the transaction must be valid, however. block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2, tx3]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) sync_blocks(self.nodes) # Add utxo to our list @@ -1204,7 +1212,7 @@ class SegWitTest(BitcoinTestFramework): # This next line will rehash the coinbase and update the merkle # root, and solve. self.update_witness_block_with_transactions(block, []) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) spend_tx = CTransaction() spend_tx.vin = [CTxIn(COutPoint(block.vtx[0].sha256, 0), b"")] @@ -1218,13 +1226,13 @@ class SegWitTest(BitcoinTestFramework): sync_blocks(self.nodes) block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) - self.test_node.test_witness_block(block2, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block2, accepted=False) # Advancing one more block should allow the spend. self.nodes[0].generate(1) block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) - self.test_node.test_witness_block(block2, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block2, accepted=True) sync_blocks(self.nodes) @@ -1245,11 +1253,11 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(self.utxo[0].nValue-1000, scriptPubKey)) tx.rehash() - self.test_node.test_transaction_acceptance(tx, with_witness=True, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=True, accepted=True) # Mine this transaction in preparation for following tests. block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) sync_blocks(self.nodes) self.utxo.pop(0) @@ -1266,19 +1274,19 @@ class SegWitTest(BitcoinTestFramework): # Too-large input value sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue+1, key) self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Too-small input value sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue-1, key) block.vtx.pop() # remove last tx self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Now try correct value sign_P2PK_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue, key) block.vtx.pop() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) prev_utxo = UTXO(tx.sha256, 0, tx.vout[0].nValue) @@ -1302,7 +1310,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) block = self.build_next_block() used_sighash_single_out_of_bounds = False @@ -1344,7 +1352,7 @@ class SegWitTest(BitcoinTestFramework): # Test the block periodically, if we're close to maxblocksize if (get_virtual_size(block) > MAX_BLOCK_BASE_SIZE - 1000): self.update_witness_block_with_transactions(block, []) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) block = self.build_next_block() if (not used_sighash_single_out_of_bounds): @@ -1352,7 +1360,7 @@ class SegWitTest(BitcoinTestFramework): # Test the transactions we've added to the block if (len(block.vtx) > 1): self.update_witness_block_with_transactions(block, []) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Now test witness version 0 P2PKH transactions pubkeyhash = hash160(pubkey) @@ -1374,7 +1382,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vin[0].scriptSig = CScript([signature, pubkey]) block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx, tx2]) - self.test_node.test_witness_block(block, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=False) # Move the signature to the witness. block.vtx.pop() @@ -1384,26 +1392,28 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) temp_utxos.pop(0) - # Update self.utxos for later tests. Just spend everything in - # temp_utxos to a corresponding entry in self.utxos + # Update self.utxos for later tests by creating two outputs + # that consolidate all the coins in temp_utxos. + output_value = sum(i.nValue for i in temp_utxos) // 2 + tx = CTransaction() index = 0 + # Just spend to our usual anyone-can-spend output + tx.vout = [CTxOut(output_value, CScript([OP_TRUE]))] * 2 for i in temp_utxos: - # Just spend to our usual anyone-can-spend output - # Use SIGHASH_SINGLE|SIGHASH_ANYONECANPAY so we can build up + # Use SIGHASH_ALL|SIGHASH_ANYONECANPAY so we can build up # the signatures as we go. tx.vin.append(CTxIn(COutPoint(i.sha256, i.n), b"")) - tx.vout.append(CTxOut(i.nValue, CScript([OP_TRUE]))) tx.wit.vtxinwit.append(CTxInWitness()) - sign_P2PK_witness_input(witness_program, tx, index, SIGHASH_SINGLE|SIGHASH_ANYONECANPAY, i.nValue, key) + sign_P2PK_witness_input(witness_program, tx, index, SIGHASH_ALL|SIGHASH_ANYONECANPAY, i.nValue, key) index += 1 block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) for i in range(len(tx.vout)): self.utxo.append(UTXO(tx.sha256, i, tx.vout[i].nValue)) @@ -1430,10 +1440,10 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() # Verify mempool acceptance and block validity - self.test_node.test_transaction_acceptance(tx, with_witness=False, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True, with_witness=segwit_activated) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True, with_witness=segwit_activated) sync_blocks(self.nodes) # Now test attempts to spend the output. @@ -1447,12 +1457,12 @@ class SegWitTest(BitcoinTestFramework): # will require a witness to spend a witness program regardless of # segwit activation. Note that older bitcoind's that are not # segwit-aware would also reject this for failing CLEANSTACK. - self.test_node.test_transaction_acceptance(spend_tx, with_witness=False, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=False, accepted=False) # Try to put the witness script in the scriptSig, should also fail. spend_tx.vin[0].scriptSig = CScript([p2wsh_pubkey, b'a']) spend_tx.rehash() - self.test_node.test_transaction_acceptance(spend_tx, with_witness=False, accepted=False) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=False, accepted=False) # Now put the witness script in the witness, should succeed after # segwit activates. @@ -1462,7 +1472,7 @@ class SegWitTest(BitcoinTestFramework): spend_tx.wit.vtxinwit[0].scriptWitness.stack = [ b'a', witness_program ] # Verify mempool acceptance - self.test_node.test_transaction_acceptance(spend_tx, with_witness=True, accepted=segwit_activated) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, spend_tx, with_witness=True, accepted=segwit_activated) block = self.build_next_block() self.update_witness_block_with_transactions(block, [spend_tx]) @@ -1470,9 +1480,9 @@ class SegWitTest(BitcoinTestFramework): # should be valid. If we're after activation, then sending this with # witnesses should be valid. if segwit_activated: - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) else: - self.test_node.test_witness_block(block, accepted=True, with_witness=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True, with_witness=False) # Update self.utxo self.utxo.pop(0) @@ -1493,7 +1503,7 @@ class SegWitTest(BitcoinTestFramework): # Restart with the new binary self.stop_node(node_id) - self.start_node(node_id, extra_args=[]) + self.start_node(node_id, extra_args=["-vbparams=segwit:0:999999999999"]) connect_nodes(self.nodes[0], node_id) sync_blocks(self.nodes) @@ -1556,7 +1566,7 @@ class SegWitTest(BitcoinTestFramework): block_1 = self.build_next_block() self.update_witness_block_with_transactions(block_1, [tx]) - self.test_node.test_witness_block(block_1, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_1, accepted=True) tx2 = CTransaction() # If we try to spend the first n-1 outputs from tx, that should be @@ -1573,7 +1583,7 @@ class SegWitTest(BitcoinTestFramework): block_2 = self.build_next_block() self.update_witness_block_with_transactions(block_2, [tx2]) - self.test_node.test_witness_block(block_2, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block_2, accepted=False) # Try dropping the last input in tx2, and add an output that has # too many sigops (contributing to legacy sigop count). @@ -1586,14 +1596,14 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() block_3 = self.build_next_block() self.update_witness_block_with_transactions(block_3, [tx2]) - self.test_node.test_witness_block(block_3, accepted=False) + test_witness_block(self.nodes[0].rpc, self.test_node, block_3, accepted=False) # If we drop the last checksig in this output, the tx should succeed. block_4 = self.build_next_block() tx2.vout[-1].scriptPubKey = CScript([OP_CHECKSIG]*(checksig_count-1)) tx2.rehash() self.update_witness_block_with_transactions(block_4, [tx2]) - self.test_node.test_witness_block(block_4, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_4, accepted=True) # Reset the tip back down for the next test sync_blocks(self.nodes) @@ -1609,7 +1619,7 @@ class SegWitTest(BitcoinTestFramework): tx2.wit.vtxinwit[-1].scriptWitness.stack = [ witness_program_justright ] tx2.rehash() self.update_witness_block_with_transactions(block_5, [tx2]) - self.test_node.test_witness_block(block_5, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block_5, accepted=True) # TODO: test p2sh sigop counting @@ -1687,7 +1697,7 @@ class SegWitTest(BitcoinTestFramework): # Confirm it in a block. block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Now try to spend it. Send it to a P2WSH output, which we'll # use in the next test. @@ -1706,11 +1716,11 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() # Should fail policy test. - self.test_node.test_transaction_acceptance(tx2, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx2, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') # But passes consensus. block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Test 2: P2WSH # Try to spend the P2WSH output created in last test. @@ -1726,11 +1736,11 @@ class SegWitTest(BitcoinTestFramework): sign_P2PK_witness_input(witness_program, tx3, 0, SIGHASH_ALL, tx2.vout[0].nValue, key) # Should fail policy test. - self.test_node.test_transaction_acceptance(tx3, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx3, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') # But passes consensus. block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx3]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Test 3: P2SH(P2WSH) # Try to spend the P2SH output created in the last test. @@ -1743,10 +1753,10 @@ class SegWitTest(BitcoinTestFramework): sign_P2PK_witness_input(witness_program, tx4, 0, SIGHASH_ALL, tx3.vout[0].nValue, key) # Should fail policy test. - self.test_node.test_transaction_acceptance(tx4, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx4, True, False, b'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx4]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) # Test 4: Uncompressed pubkeys should still be valid in non-segwit # transactions. @@ -1758,10 +1768,10 @@ class SegWitTest(BitcoinTestFramework): tx5.vin[0].scriptSig = CScript([signature, pubkey]) tx5.rehash() # Should pass policy and consensus. - self.test_node.test_transaction_acceptance(tx5, True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx5, True, True) block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx5]) - self.test_node.test_witness_block(block, accepted=True) + test_witness_block(self.nodes[0].rpc, self.test_node, block, accepted=True) self.utxo.append(UTXO(tx5.sha256, 0, tx5.vout[0].nValue)) def test_non_standard_witness(self): @@ -1791,7 +1801,7 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(outputvalue, CScript([OP_HASH160, p2sh, OP_EQUAL]))) tx.rehash() txid = tx.sha256 - self.test_node.test_transaction_acceptance(tx, with_witness=False, accepted=True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, tx, with_witness=False, accepted=True) self.nodes[0].generate(1) sync_blocks(self.nodes) @@ -1816,45 +1826,45 @@ class SegWitTest(BitcoinTestFramework): # Testing native P2WSH # Witness stack size, excluding witnessScript, over 100 is non-standard p2wsh_txs[0].wit.vtxinwit[0].scriptWitness.stack = [pad] * 101 + [scripts[0]] - self.std_node.test_transaction_acceptance(p2wsh_txs[0], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[0], True, False, b'bad-witness-nonstandard') # Non-standard nodes should accept - self.test_node.test_transaction_acceptance(p2wsh_txs[0], True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[0], True, True) # Stack element size over 80 bytes is non-standard p2wsh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 81] * 100 + [scripts[1]] - self.std_node.test_transaction_acceptance(p2wsh_txs[1], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[1], True, False, b'bad-witness-nonstandard') # Non-standard nodes should accept - self.test_node.test_transaction_acceptance(p2wsh_txs[1], True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[1], True, True) # Standard nodes should accept if element size is not over 80 bytes p2wsh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 80] * 100 + [scripts[1]] - self.std_node.test_transaction_acceptance(p2wsh_txs[1], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[1], True, True) # witnessScript size at 3600 bytes is standard p2wsh_txs[2].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, scripts[2]] - self.test_node.test_transaction_acceptance(p2wsh_txs[2], True, True) - self.std_node.test_transaction_acceptance(p2wsh_txs[2], True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[2], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[2], True, True) # witnessScript size at 3601 bytes is non-standard p2wsh_txs[3].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, pad, scripts[3]] - self.std_node.test_transaction_acceptance(p2wsh_txs[3], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2wsh_txs[3], True, False, b'bad-witness-nonstandard') # Non-standard nodes should accept - self.test_node.test_transaction_acceptance(p2wsh_txs[3], True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2wsh_txs[3], True, True) # Repeating the same tests with P2SH-P2WSH p2sh_txs[0].wit.vtxinwit[0].scriptWitness.stack = [pad] * 101 + [scripts[0]] - self.std_node.test_transaction_acceptance(p2sh_txs[0], True, False, b'bad-witness-nonstandard') - self.test_node.test_transaction_acceptance(p2sh_txs[0], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[0], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[0], True, True) p2sh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 81] * 100 + [scripts[1]] - self.std_node.test_transaction_acceptance(p2sh_txs[1], True, False, b'bad-witness-nonstandard') - self.test_node.test_transaction_acceptance(p2sh_txs[1], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[1], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[1], True, True) p2sh_txs[1].wit.vtxinwit[0].scriptWitness.stack = [pad * 80] * 100 + [scripts[1]] - self.std_node.test_transaction_acceptance(p2sh_txs[1], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[1], True, True) p2sh_txs[2].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, scripts[2]] - self.test_node.test_transaction_acceptance(p2sh_txs[2], True, True) - self.std_node.test_transaction_acceptance(p2sh_txs[2], True, True) + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[2], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[2], True, True) p2sh_txs[3].wit.vtxinwit[0].scriptWitness.stack = [pad, pad, pad, scripts[3]] - self.std_node.test_transaction_acceptance(p2sh_txs[3], True, False, b'bad-witness-nonstandard') - self.test_node.test_transaction_acceptance(p2sh_txs[3], True, True) + test_transaction_acceptance(self.nodes[1].rpc, self.std_node, p2sh_txs[3], True, False, b'bad-witness-nonstandard') + test_transaction_acceptance(self.nodes[0].rpc, self.test_node, p2sh_txs[3], True, True) self.nodes[0].generate(1) # Mine and clean up the mempool of non-standard node # Valid but non-standard transactions in a block should be accepted by standard node @@ -1867,21 +1877,14 @@ class SegWitTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections and start up the network thread. - self.test_node = TestNode() # sets NODE_WITNESS|NODE_NETWORK - self.old_node = TestNode() # only NODE_NETWORK - self.std_node = TestNode() # for testing node1 (fRequireStandard=true) - - self.p2p_connections = [self.test_node, self.old_node] - - self.connections = [] - self.connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node, services=NODE_NETWORK|NODE_WITNESS)) - self.connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.old_node, services=NODE_NETWORK)) - self.connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], self.std_node, services=NODE_NETWORK|NODE_WITNESS)) - self.test_node.add_connection(self.connections[0]) - self.old_node.add_connection(self.connections[1]) - self.std_node.add_connection(self.connections[2]) - - NetworkThread().start() # Start up network handling in another thread + # self.test_node sets NODE_WITNESS|NODE_NETWORK + self.test_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + # self.old_node sets only NODE_NETWORK + self.old_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_NETWORK) + # self.std_node is for testing node1 (fRequireStandard=true) + self.std_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + + network_thread_start() # Keep a place to store utxo's that can be used in later tests self.utxo = [] diff --git a/test/functional/p2p-timeouts.py b/test/functional/p2p-timeouts.py index 51d4769efc..6d21095cc6 100755 --- a/test/functional/p2p-timeouts.py +++ b/test/functional/p2p-timeouts.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various net timeouts. @@ -27,8 +27,8 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -class TestNode(NodeConnCB): - def on_version(self, conn, message): +class TestNode(P2PInterface): + def on_version(self, message): # Don't send a verack in response pass @@ -39,46 +39,37 @@ class TimeoutsTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections and start up the network thread. - self.no_verack_node = TestNode() # never send verack - self.no_version_node = TestNode() # never send version (just ping) - self.no_send_node = TestNode() # never send anything + no_verack_node = self.nodes[0].add_p2p_connection(TestNode()) + no_version_node = self.nodes[0].add_p2p_connection(TestNode(), send_version=False) + no_send_node = self.nodes[0].add_p2p_connection(TestNode(), send_version=False) - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.no_verack_node)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.no_version_node, send_version=False)) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.no_send_node, send_version=False)) - self.no_verack_node.add_connection(connections[0]) - self.no_version_node.add_connection(connections[1]) - self.no_send_node.add_connection(connections[2]) - - NetworkThread().start() # Start up network handling in another thread + network_thread_start() sleep(1) - assert(self.no_verack_node.connected) - assert(self.no_version_node.connected) - assert(self.no_send_node.connected) + assert no_verack_node.connected + assert no_version_node.connected + assert no_send_node.connected - ping_msg = msg_ping() - connections[0].send_message(ping_msg) - connections[1].send_message(ping_msg) + no_verack_node.send_message(msg_ping()) + no_version_node.send_message(msg_ping()) sleep(30) - assert "version" in self.no_verack_node.last_message + assert "version" in no_verack_node.last_message - assert(self.no_verack_node.connected) - assert(self.no_version_node.connected) - assert(self.no_send_node.connected) + assert no_verack_node.connected + assert no_version_node.connected + assert no_send_node.connected - connections[0].send_message(ping_msg) - connections[1].send_message(ping_msg) + no_verack_node.send_message(msg_ping()) + no_version_node.send_message(msg_ping()) sleep(31) - assert(not self.no_verack_node.connected) - assert(not self.no_version_node.connected) - assert(not self.no_send_node.connected) + assert not no_verack_node.connected + assert not no_version_node.connected + assert not no_send_node.connected if __name__ == '__main__': TimeoutsTest().main() diff --git a/test/functional/p2p-versionbits-warning.py b/test/functional/p2p-versionbits-warning.py index f9bef2580a..0bfe94622f 100755 --- a/test/functional/p2p-versionbits-warning.py +++ b/test/functional/p2p-versionbits-warning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test version bits warning system. @@ -23,8 +23,8 @@ WARN_UNKNOWN_RULES_MINED = "Unknown block versions being mined! It's possible un WARN_UNKNOWN_RULES_ACTIVE = "unknown new rules activated (versionbit {})".format(VB_UNKNOWN_BIT) VB_PATTERN = re.compile("^Warning.*versionbit") -class TestNode(NodeConnCB): - def on_inv(self, conn, message): +class TestNode(P2PInterface): + def on_inv(self, message): pass class VersionBitsWarningTest(BitcoinTestFramework): @@ -64,16 +64,12 @@ class VersionBitsWarningTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connection and start up the network thread. - test_node = TestNode() + self.nodes[0].add_p2p_connection(TestNode()) - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node)) - test_node.add_connection(connections[0]) - - NetworkThread().start() # Start up network handling in another thread + network_thread_start() # Test logic begins here - test_node.wait_for_verack() + self.nodes[0].p2p.wait_for_verack() # 1. Have the node mine one period worth of blocks self.nodes[0].generate(VB_PERIOD) @@ -81,7 +77,7 @@ class VersionBitsWarningTest(BitcoinTestFramework): # 2. Now build one period of blocks on the tip, with < VB_THRESHOLD # blocks signaling some unknown bit. nVersion = VB_TOP_BITS | (1<<VB_UNKNOWN_BIT) - self.send_blocks_with_version(test_node, VB_THRESHOLD-1, nVersion) + self.send_blocks_with_version(self.nodes[0].p2p, VB_THRESHOLD-1, nVersion) # Fill rest of period with regular version blocks self.nodes[0].generate(VB_PERIOD - VB_THRESHOLD + 1) @@ -92,7 +88,7 @@ class VersionBitsWarningTest(BitcoinTestFramework): # 3. Now build one period of blocks with >= VB_THRESHOLD blocks signaling # some unknown bit - self.send_blocks_with_version(test_node, VB_THRESHOLD, nVersion) + self.send_blocks_with_version(self.nodes[0].p2p, VB_THRESHOLD, nVersion) self.nodes[0].generate(VB_PERIOD - VB_THRESHOLD) # Might not get a versionbits-related alert yet, as we should # have gotten a different alert due to more than 51/100 blocks diff --git a/test/functional/preciousblock.py b/test/functional/preciousblock.py index 1466f901c0..960cd0ad12 100755 --- a/test/functional/preciousblock.py +++ b/test/functional/preciousblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the preciousblock RPC.""" diff --git a/test/functional/prioritise_transaction.py b/test/functional/prioritise_transaction.py index bb56db9b40..57954ce321 100755 --- a/test/functional/prioritise_transaction.py +++ b/test/functional/prioritise_transaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the prioritisetransaction mining RPC.""" @@ -15,6 +15,25 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.extra_args = [["-printpriority=1"], ["-printpriority=1"]] def run_test(self): + # Test `prioritisetransaction` required parameters + assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction) + assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction, '') + assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction, '', 0) + + # Test `prioritisetransaction` invalid extra parameters + assert_raises_rpc_error(-1, "prioritisetransaction", self.nodes[0].prioritisetransaction, '', 0, 0, 0) + + # Test `prioritisetransaction` invalid `txid` + assert_raises_rpc_error(-1, "txid must be hexadecimal string", self.nodes[0].prioritisetransaction, txid='foo', fee_delta=0) + + # Test `prioritisetransaction` invalid `dummy` + txid = '1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000' + assert_raises_rpc_error(-1, "JSON value is not a number as expected", self.nodes[0].prioritisetransaction, txid, 'foo', 0) + assert_raises_rpc_error(-8, "Priority is no longer supported, dummy argument to prioritisetransaction must be 0.", self.nodes[0].prioritisetransaction, txid, 1, 0) + + # Test `prioritisetransaction` invalid `fee_delta` + assert_raises_rpc_error(-1, "JSON value is not an integer as expected", self.nodes[0].prioritisetransaction, txid=txid, fee_delta='foo') + self.txouts = gen_return_txouts() self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] diff --git a/test/functional/proxy_test.py b/test/functional/proxy_test.py index 81b99d1bf4..2eb1be47a5 100755 --- a/test/functional/proxy_test.py +++ b/test/functional/proxy_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind with different proxy configuration. diff --git a/test/functional/pruning.py b/test/functional/pruning.py index 0101f61185..49ad7f838c 100755 --- a/test/functional/pruning.py +++ b/test/functional/pruning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the pruning code. diff --git a/test/functional/rawtransactions.py b/test/functional/rawtransactions.py index 2777cb9693..fe749adb49 100755 --- a/test/functional/rawtransactions.py +++ b/test/functional/rawtransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the rawtransaction RPCs. @@ -15,11 +15,31 @@ Test the following RPCs: from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * + +class multidict(dict): + """Dictionary that allows duplicate keys. + + Constructed with a list of (key, value) tuples. When dumped by the json module, + will output invalid json with repeated keys, eg: + >>> json.dumps(multidict([(1,2),(1,2)]) + '{"1": 2, "1": 2}' + + Used to test calls to rpc methods with repeated keys in the json object.""" + + def __init__(self, x): + dict.__init__(self, x) + self.x = x + + def items(self): + return self.x + + # Create one-input, one-output, no-fee transaction: class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [["-addresstype=legacy"], ["-addresstype=legacy"], ["-addresstype=legacy"]] def setup_network(self, split=False): super().setup_network() @@ -39,6 +59,41 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].generate(5) self.sync_all() + # Test `createrawtransaction` required parameters + assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction) + assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction, []) + + # Test `createrawtransaction` invalid extra parameters + assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction, [], {}, 0, False, 'foo') + + # Test `createrawtransaction` invalid `inputs` + txid = '1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000' + assert_raises_rpc_error(-3, "Expected type array", self.nodes[0].createrawtransaction, 'foo', {}) + assert_raises_rpc_error(-1, "JSON value is not an object as expected", self.nodes[0].createrawtransaction, ['foo'], {}) + assert_raises_rpc_error(-8, "txid must be hexadecimal string", self.nodes[0].createrawtransaction, [{}], {}) + assert_raises_rpc_error(-8, "txid must be hexadecimal string", self.nodes[0].createrawtransaction, [{'txid': 'foo'}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", self.nodes[0].createrawtransaction, [{'txid': txid}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", self.nodes[0].createrawtransaction, [{'txid': txid, 'vout': 'foo'}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, vout must be positive", self.nodes[0].createrawtransaction, [{'txid': txid, 'vout': -1}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, sequence number is out of range", self.nodes[0].createrawtransaction, [{'txid': txid, 'vout': 0, 'sequence': -1}], {}) + + # Test `createrawtransaction` invalid `outputs` + address = self.nodes[0].getnewaddress() + assert_raises_rpc_error(-3, "Expected type object", self.nodes[0].createrawtransaction, [], 'foo') + assert_raises_rpc_error(-8, "Data must be hexadecimal string", self.nodes[0].createrawtransaction, [], {'data': 'foo'}) + assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].createrawtransaction, [], {'foo': 0}) + assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].createrawtransaction, [], {address: 'foo'}) + assert_raises_rpc_error(-3, "Amount out of range", self.nodes[0].createrawtransaction, [], {address: -1}) + assert_raises_rpc_error(-8, "Invalid parameter, duplicated address: %s" % address, self.nodes[0].createrawtransaction, [], multidict([(address, 1), (address, 1)])) + + # Test `createrawtransaction` invalid `locktime` + assert_raises_rpc_error(-3, "Expected type number", self.nodes[0].createrawtransaction, [], {}, 'foo') + assert_raises_rpc_error(-8, "Invalid parameter, locktime out of range", self.nodes[0].createrawtransaction, [], {}, -1) + assert_raises_rpc_error(-8, "Invalid parameter, locktime out of range", self.nodes[0].createrawtransaction, [], {}, 4294967296) + + # Test `createrawtransaction` invalid `replaceable` + assert_raises_rpc_error(-3, "Expected type bool", self.nodes[0].createrawtransaction, [], {}, 0, 'foo') + ######################################### # sendrawtransaction with missing input # ######################################### @@ -50,6 +105,36 @@ class RawTransactionsTest(BitcoinTestFramework): # This will raise an exception since there are missing inputs assert_raises_rpc_error(-25, "Missing inputs", self.nodes[2].sendrawtransaction, rawtx['hex']) + ##################################### + # getrawtransaction with block hash # + ##################################### + + # make a tx by sending then generate 2 blocks; block1 has the tx in it + tx = self.nodes[2].sendtoaddress(self.nodes[1].getnewaddress(), 1) + block1, block2 = self.nodes[2].generate(2) + self.sync_all() + # We should be able to get the raw transaction by providing the correct block + gottx = self.nodes[0].getrawtransaction(tx, True, block1) + assert_equal(gottx['txid'], tx) + assert_equal(gottx['in_active_chain'], True) + # We should not have the 'in_active_chain' flag when we don't provide a block + gottx = self.nodes[0].getrawtransaction(tx, True) + assert_equal(gottx['txid'], tx) + assert 'in_active_chain' not in gottx + # We should not get the tx if we provide an unrelated block + assert_raises_rpc_error(-5, "No such transaction found", self.nodes[0].getrawtransaction, tx, True, block2) + # An invalid block hash should raise the correct errors + assert_raises_rpc_error(-8, "parameter 3 must be hexadecimal", self.nodes[0].getrawtransaction, tx, True, True) + assert_raises_rpc_error(-8, "parameter 3 must be hexadecimal", self.nodes[0].getrawtransaction, tx, True, "foobar") + assert_raises_rpc_error(-8, "parameter 3 must be of length 64", self.nodes[0].getrawtransaction, tx, True, "abcd1234") + assert_raises_rpc_error(-5, "Block hash not found", self.nodes[0].getrawtransaction, tx, True, "0000000000000000000000000000000000000000000000000000000000000000") + # Undo the blocks and check in_active_chain + self.nodes[0].invalidateblock(block1) + gottx = self.nodes[0].getrawtransaction(txid=tx, verbose=True, blockhash=block1) + assert_equal(gottx['in_active_chain'], False) + self.nodes[0].reconsiderblock(block1) + assert_equal(self.nodes[0].getbestblockhash(), block2) + ######################### # RAW TX MULTISIG TESTS # ######################### @@ -105,7 +190,7 @@ class RawTransactionsTest(BitcoinTestFramework): break bal = self.nodes[0].getbalance() - inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex']}] + inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "amount" : vout['value']}] outputs = { self.nodes[0].getnewaddress() : 2.19 } rawTx = self.nodes[2].createrawtransaction(inputs, outputs) rawTxPartialSigned = self.nodes[1].signrawtransaction(rawTx, inputs) @@ -150,7 +235,7 @@ class RawTransactionsTest(BitcoinTestFramework): break bal = self.nodes[0].getbalance() - inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "redeemScript" : mSigObjValid['hex']}] + inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "redeemScript" : mSigObjValid['hex'], "amount" : vout['value']}] outputs = { self.nodes[0].getnewaddress() : 2.19 } rawTx2 = self.nodes[2].createrawtransaction(inputs, outputs) rawTxPartialSigned1 = self.nodes[1].signrawtransaction(rawTx2, inputs) @@ -169,6 +254,17 @@ class RawTransactionsTest(BitcoinTestFramework): self.sync_all() assert_equal(self.nodes[0].getbalance(), bal+Decimal('50.00000000')+Decimal('2.19000000')) #block reward + tx + # decoderawtransaction tests + # witness transaction + encrawtx = "010000000001010000000000000072c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000000ffffffff0100e1f50500000000000000000000" + decrawtx = self.nodes[0].decoderawtransaction(encrawtx, True) # decode as witness transaction + assert_equal(decrawtx['vout'][0]['value'], Decimal('1.00000000')) + assert_raises_rpc_error(-22, 'TX decode failed', self.nodes[0].decoderawtransaction, encrawtx, False) # force decode as non-witness transaction + # non-witness transaction + encrawtx = "01000000010000000000000072c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000000ffffffff0100e1f505000000000000000000" + decrawtx = self.nodes[0].decoderawtransaction(encrawtx, False) # decode as non-witness transaction + assert_equal(decrawtx['vout'][0]['value'], Decimal('1.00000000')) + # getrawtransaction tests # 1. valid parameters - only supply txid txHash = rawTx["hash"] @@ -188,13 +284,13 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(self.nodes[0].getrawtransaction(txHash, True)["hex"], rawTxSigned['hex']) # 6. invalid parameters - supply txid and string "Flase" - assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, "Flase") + assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txHash, "Flase") # 7. invalid parameters - supply txid and empty array - assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, []) + assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txHash, []) # 8. invalid parameters - supply txid and empty dict - assert_raises_rpc_error(-3,"Invalid type", self.nodes[0].getrawtransaction, txHash, {}) + assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txHash, {}) inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 1000}] outputs = { self.nodes[0].getnewaddress() : 1 } diff --git a/test/functional/receivedby.py b/test/functional/receivedby.py index db6fc86b82..1f2b3c8aa7 100755 --- a/test/functional/receivedby.py +++ b/test/functional/receivedby.py @@ -1,99 +1,85 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listreceivedbyaddress RPC.""" +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -def get_sub_array_from_array(object_array, to_match): - ''' - Finds and returns a sub array from an array of arrays. - to_match should be a unique idetifier of a sub array - ''' - for item in object_array: - all_match = True - for key,value in to_match.items(): - if item[key] != value: - all_match = False - if not all_match: - continue - return item - return [] +from test_framework.util import (assert_array_result, + assert_equal, + assert_raises_rpc_error, + ) class ReceivedByTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.enable_mocktime() def run_test(self): - ''' - listreceivedbyaddress Test - ''' + # Generate block to get out of IBD + self.nodes[0].generate(1) + + self.log.info("listreceivedbyaddress Test") + # Send from node 0 to 1 addr = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(addr, 0.1) self.sync_all() - #Check not listed in listreceivedbyaddress because has 0 confirmations + # Check not listed in listreceivedbyaddress because has 0 confirmations assert_array_result(self.nodes[1].listreceivedbyaddress(), - {"address":addr}, - { }, - True) - #Bury Tx under 10 block so it will be returned by listreceivedbyaddress + {"address": addr}, + {}, + True) + # Bury Tx under 10 block so it will be returned by listreceivedbyaddress self.nodes[1].generate(10) self.sync_all() assert_array_result(self.nodes[1].listreceivedbyaddress(), - {"address":addr}, - {"address":addr, "account":"", "amount":Decimal("0.1"), "confirmations":10, "txids":[txid,]}) - #With min confidence < 10 + {"address": addr}, + {"address": addr, "account": "", "amount": Decimal("0.1"), "confirmations": 10, "txids": [txid, ]}) + # With min confidence < 10 assert_array_result(self.nodes[1].listreceivedbyaddress(5), - {"address":addr}, - {"address":addr, "account":"", "amount":Decimal("0.1"), "confirmations":10, "txids":[txid,]}) - #With min confidence > 10, should not find Tx - assert_array_result(self.nodes[1].listreceivedbyaddress(11),{"address":addr},{ },True) + {"address": addr}, + {"address": addr, "account": "", "amount": Decimal("0.1"), "confirmations": 10, "txids": [txid, ]}) + # With min confidence > 10, should not find Tx + assert_array_result(self.nodes[1].listreceivedbyaddress(11), {"address": addr}, {}, True) - #Empty Tx + # Empty Tx addr = self.nodes[1].getnewaddress() - assert_array_result(self.nodes[1].listreceivedbyaddress(0,True), - {"address":addr}, - {"address":addr, "account":"", "amount":0, "confirmations":0, "txids":[]}) + assert_array_result(self.nodes[1].listreceivedbyaddress(0, True), + {"address": addr}, + {"address": addr, "account": "", "amount": 0, "confirmations": 0, "txids": []}) + + self.log.info("getreceivedbyaddress Test") - ''' - getreceivedbyaddress Test - ''' # Send from node 0 to 1 addr = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(addr, 0.1) self.sync_all() - #Check balance is 0 because of 0 confirmations + # Check balance is 0 because of 0 confirmations balance = self.nodes[1].getreceivedbyaddress(addr) - if balance != Decimal("0.0"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.0")) - #Check balance is 0.1 - balance = self.nodes[1].getreceivedbyaddress(addr,0) - if balance != Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + # Check balance is 0.1 + balance = self.nodes[1].getreceivedbyaddress(addr, 0) + assert_equal(balance, Decimal("0.1")) - #Bury Tx under 10 block so it will be returned by the default getreceivedbyaddress + # Bury Tx under 10 block so it will be returned by the default getreceivedbyaddress self.nodes[1].generate(10) self.sync_all() balance = self.nodes[1].getreceivedbyaddress(addr) - if balance != Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.1")) + + # Trying to getreceivedby for an address the wallet doesn't own should return an error + assert_raises_rpc_error(-4, "Address not found in wallet", self.nodes[0].getreceivedbyaddress, addr) + + self.log.info("listreceivedbyaccount + getreceivedbyaccount Test") - ''' - listreceivedbyaccount + getreceivedbyaccount Test - ''' - #set pre-state + # set pre-state addrArr = self.nodes[1].getnewaddress() account = self.nodes[1].getaccount(addrArr) - received_by_account_json = get_sub_array_from_array(self.nodes[1].listreceivedbyaccount(),{"account":account}) - if len(received_by_account_json) == 0: - raise AssertionError("No accounts found in node") + received_by_account_json = [r for r in self.nodes[1].listreceivedbyaccount() if r["account"] == account][0] balance_by_account = self.nodes[1].getreceivedbyaccount(account) txid = self.nodes[0].sendtoaddress(addr, 0.1) @@ -101,40 +87,34 @@ class ReceivedByTest(BitcoinTestFramework): # listreceivedbyaccount should return received_by_account_json because of 0 confirmations assert_array_result(self.nodes[1].listreceivedbyaccount(), - {"account":account}, - received_by_account_json) + {"account": account}, + received_by_account_json) # getreceivedbyaddress should return same balance because of 0 confirmations balance = self.nodes[1].getreceivedbyaccount(account) - if balance != balance_by_account: - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, balance_by_account) self.nodes[1].generate(10) self.sync_all() # listreceivedbyaccount should return updated account balance assert_array_result(self.nodes[1].listreceivedbyaccount(), - {"account":account}, - {"account":received_by_account_json["account"], "amount":(received_by_account_json["amount"] + Decimal("0.1"))}) + {"account": account}, + {"account": received_by_account_json["account"], "amount": (received_by_account_json["amount"] + Decimal("0.1"))}) # getreceivedbyaddress should return updates balance balance = self.nodes[1].getreceivedbyaccount(account) - if balance != balance_by_account + Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, balance_by_account + Decimal("0.1")) - #Create a new account named "mynewaccount" that has a 0 balance + # Create a new account named "mynewaccount" that has a 0 balance self.nodes[1].getaccountaddress("mynewaccount") - received_by_account_json = get_sub_array_from_array(self.nodes[1].listreceivedbyaccount(0,True),{"account":"mynewaccount"}) - if len(received_by_account_json) == 0: - raise AssertionError("No accounts found in node") + received_by_account_json = [r for r in self.nodes[1].listreceivedbyaccount(0, True) if r["account"] == "mynewaccount"][0] # Test includeempty of listreceivedbyaccount - if received_by_account_json["amount"] != Decimal("0.0"): - raise AssertionError("Wrong balance returned by listreceivedbyaccount, %0.2f"%(received_by_account_json["amount"])) + assert_equal(received_by_account_json["amount"], Decimal("0.0")) # Test getreceivedbyaccount for 0 amount accounts balance = self.nodes[1].getreceivedbyaccount("mynewaccount") - if balance != Decimal("0.0"): - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.0")) if __name__ == '__main__': ReceivedByTest().main() diff --git a/test/functional/reindex.py b/test/functional/reindex.py index 1f684a1afe..ac67e6e9ba 100755 --- a/test/functional/reindex.py +++ b/test/functional/reindex.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with -reindex and -reindex-chainstate options. diff --git a/test/functional/replace-by-fee.py b/test/functional/replace-by-fee.py index 269d57775c..6b7ab0f43e 100755 --- a/test/functional/replace-by-fee.py +++ b/test/functional/replace-by-fee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" @@ -72,8 +72,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): ["-mempoolreplacement=0"]] def run_test(self): + # Leave IBD + self.nodes[0].generate(1) + make_utxo(self.nodes[0], 1*COIN) + # Ensure nodes are synced + self.sync_all() + self.log.info("Running test simple doublespend...") self.test_simple_doublespend() @@ -110,13 +116,18 @@ class ReplaceByFeeTest(BitcoinTestFramework): """Simple doublespend""" tx0_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) + # make_utxo may have generated a bunch of blocks, so we need to sync + # before we can spend the coins generated, or else the resulting + # transactions might not be accepted by our peers. + self.sync_all() + tx1a = CTransaction() tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] tx1a.vout = [CTxOut(1*COIN, CScript([b'a']))] tx1a_hex = txToHex(tx1a) tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, True) - self.sync_all([self.nodes]) + self.sync_all() # Should fail because we haven't changed the fee tx1b = CTransaction() diff --git a/test/functional/rest.py b/test/functional/rest.py index 437111a4d7..9006e27cbe 100755 --- a/test/functional/rest.py +++ b/test/functional/rest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the REST API.""" diff --git a/test/functional/rpcbind_test.py b/test/functional/rpcbind_test.py index 0e8c3fa209..05433c7e24 100755 --- a/test/functional/rpcbind_test.py +++ b/test/functional/rpcbind_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with the -rpcbind and -rpcallowip options.""" diff --git a/test/functional/rpcnamedargs.py b/test/functional/rpcnamedargs.py index c47212bddb..97bee39614 100755 --- a/test/functional/rpcnamedargs.py +++ b/test/functional/rpcnamedargs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test using named arguments for RPCs.""" diff --git a/test/functional/segwit.py b/test/functional/segwit.py index f03eda5bb2..7d5c760ad9 100755 --- a/test/functional/segwit.py +++ b/test/functional/segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the SegWit changeover logic.""" @@ -39,9 +39,10 @@ class SegWitTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 - self.extra_args = [["-walletprematurewitness", "-rpcserialversion=0"], - ["-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-rpcserialversion=1"], - ["-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness"]] + # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. + self.extra_args = [["-walletprematurewitness", "-rpcserialversion=0", "-vbparams=segwit:0:999999999999", "-addresstype=legacy"], + ["-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-rpcserialversion=1", "-vbparams=segwit:0:999999999999", "-addresstype=legacy"], + ["-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-vbparams=segwit:0:999999999999", "-addresstype=legacy"]] def setup_network(self): super().setup_network() @@ -96,9 +97,9 @@ class SegWitTest(BitcoinTestFramework): self.pubkey.append(self.nodes[i].validateaddress(newaddress)["pubkey"]) multiaddress = self.nodes[i].addmultisigaddress(1, [self.pubkey[-1]]) multiscript = CScript([OP_1, hex_str_to_bytes(self.pubkey[-1]), OP_1, OP_CHECKMULTISIG]) - p2sh_addr = self.nodes[i].addwitnessaddress(newaddress, True) + p2sh_addr = self.nodes[i].addwitnessaddress(newaddress) bip173_addr = self.nodes[i].addwitnessaddress(newaddress, False) - p2sh_ms_addr = self.nodes[i].addwitnessaddress(multiaddress, True) + p2sh_ms_addr = self.nodes[i].addwitnessaddress(multiaddress) bip173_ms_addr = self.nodes[i].addwitnessaddress(multiaddress, False) assert_equal(p2sh_addr, key_to_p2sh_p2wpkh(self.pubkey[-1])) assert_equal(bip173_addr, key_to_p2wpkh(self.pubkey[-1])) @@ -317,8 +318,10 @@ class SegWitTest(BitcoinTestFramework): [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) # normal P2PKH and P2PK with compressed keys should always be spendable spendable_anytime.extend([p2pkh, p2pk]) - # P2SH_P2PK, P2SH_P2PKH, and witness with compressed keys are spendable after direct importaddress - spendable_after_importaddress.extend([p2wpkh, p2sh_p2wpkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) + # P2SH_P2PK, P2SH_P2PKH with compressed keys are spendable after direct importaddress + spendable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) + # P2WPKH and P2SH_P2WPKH with compressed keys should always be spendable + spendable_anytime.extend([p2wpkh, p2sh_p2wpkh]) for i in uncompressed_spendable_address: v = self.nodes[0].validateaddress(i) @@ -334,7 +337,7 @@ class SegWitTest(BitcoinTestFramework): spendable_anytime.extend([p2pkh, p2pk]) # P2SH_P2PK and P2SH_P2PKH are spendable after direct importaddress spendable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh]) - # witness with uncompressed keys are never seen + # Witness output types with uncompressed keys are never seen unseen_anytime.extend([p2wpkh, p2sh_p2wpkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) for i in compressed_solvable_address: @@ -345,10 +348,10 @@ class SegWitTest(BitcoinTestFramework): solvable_after_importaddress.extend([bare, p2sh, p2wsh, p2sh_p2wsh]) else: [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) - # normal P2PKH and P2PK with compressed keys should always be seen - solvable_anytime.extend([p2pkh, p2pk]) - # P2SH_P2PK, P2SH_P2PKH, and witness with compressed keys are seen after direct importaddress - solvable_after_importaddress.extend([p2wpkh, p2sh_p2wpkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) + # normal P2PKH, P2PK, P2WPKH and P2SH_P2WPKH with compressed keys should always be seen + solvable_anytime.extend([p2pkh, p2pk, p2wpkh, p2sh_p2wpkh]) + # P2SH_P2PK, P2SH_P2PKH with compressed keys are seen after direct importaddress + solvable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) for i in uncompressed_solvable_address: v = self.nodes[0].validateaddress(i) @@ -364,7 +367,7 @@ class SegWitTest(BitcoinTestFramework): solvable_anytime.extend([p2pkh, p2pk]) # P2SH_P2PK, P2SH_P2PKH with uncompressed keys are seen after direct importaddress solvable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh]) - # witness with uncompressed keys are never seen + # Witness output types with uncompressed keys are never seen unseen_anytime.extend([p2wpkh, p2sh_p2wpkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) op1 = CScript([OP_1]) @@ -457,6 +460,8 @@ class SegWitTest(BitcoinTestFramework): spendable_after_addwitnessaddress = [] # These outputs should be seen after importaddress solvable_after_addwitnessaddress=[] # These outputs should be seen after importaddress but not spendable unseen_anytime = [] # These outputs should never be seen + solvable_anytime = [] # These outputs should be solvable after importpubkey + unseen_anytime = [] # These outputs should never be seen uncompressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [uncompressed_spendable_address[0], compressed_spendable_address[0]])) uncompressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [uncompressed_spendable_address[0], uncompressed_spendable_address[0]])) @@ -475,9 +480,8 @@ class SegWitTest(BitcoinTestFramework): premature_witaddress.append(script_to_p2sh(p2wsh)) else: [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) - # P2WPKH, P2SH_P2WPKH are spendable after addwitnessaddress - spendable_after_addwitnessaddress.extend([p2wpkh, p2sh_p2wpkh]) - premature_witaddress.append(script_to_p2sh(p2wpkh)) + # P2WPKH, P2SH_P2WPKH are always spendable + spendable_anytime.extend([p2wpkh, p2sh_p2wpkh]) for i in uncompressed_spendable_address + uncompressed_solvable_address: v = self.nodes[0].validateaddress(i) @@ -499,10 +503,11 @@ class SegWitTest(BitcoinTestFramework): premature_witaddress.append(script_to_p2sh(p2wsh)) else: [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) - # P2SH_P2PK, P2SH_P2PKH with compressed keys are seen after addwitnessaddress - solvable_after_addwitnessaddress.extend([p2wpkh, p2sh_p2wpkh]) - premature_witaddress.append(script_to_p2sh(p2wpkh)) + # P2SH_P2PK, P2SH_P2PKH with compressed keys are always solvable + solvable_anytime.extend([p2wpkh, p2sh_p2wpkh]) + self.mine_and_test_listunspent(spendable_anytime, 2) + self.mine_and_test_listunspent(solvable_anytime, 1) self.mine_and_test_listunspent(spendable_after_addwitnessaddress + solvable_after_addwitnessaddress + unseen_anytime, 0) # addwitnessaddress should refuse to return a witness address if an uncompressed key is used @@ -519,8 +524,8 @@ class SegWitTest(BitcoinTestFramework): witaddress = self.nodes[0].addwitnessaddress(i) assert_equal(witaddress, self.nodes[0].addwitnessaddress(witaddress)) - spendable_txid.append(self.mine_and_test_listunspent(spendable_after_addwitnessaddress, 2)) - solvable_txid.append(self.mine_and_test_listunspent(solvable_after_addwitnessaddress, 1)) + spendable_txid.append(self.mine_and_test_listunspent(spendable_after_addwitnessaddress + spendable_anytime, 2)) + solvable_txid.append(self.mine_and_test_listunspent(solvable_after_addwitnessaddress + solvable_anytime, 1)) self.mine_and_test_listunspent(unseen_anytime, 0) # Check that createrawtransaction/decoderawtransaction with non-v0 Bech32 works diff --git a/test/functional/sendheaders.py b/test/functional/sendheaders.py index fe577dc20a..8869aeaaea 100755 --- a/test/functional/sendheaders.py +++ b/test/functional/sendheaders.py @@ -1,14 +1,27 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of headers messages to announce blocks. -Setup: +Setup: -- Two nodes, two p2p connections to node0. One p2p connection should only ever - receive inv's (omitted from testing description below, this is our control). - Second node is used for creating reorgs. +- Two nodes: + - node0 is the node-under-test. We create two p2p connections to it. The + first p2p connection is a control and should only ever receive inv's. The + second p2p connection tests the headers sending logic. + - node1 is used to create reorgs. + +test_null_locators +================== + +Sends two getheaders requests with null locator values. First request's hashstop +value refers to validated block, while second request's hashstop value refers to +a block which hasn't been validated. Verifies only the first request returns +headers. + +test_nonnull_locators +===================== Part 1: No headers announcements before "sendheaders" a. node mines a block [expect: inv] @@ -72,159 +85,197 @@ d. Announce 49 headers that don't connect. e. Announce one more that doesn't connect. Expect: disconnect. """ - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * from test_framework.blocktools import create_block, create_coinbase +from test_framework.mininode import ( + CBlockHeader, + CInv, + NODE_WITNESS, + network_thread_start, + P2PInterface, + mininode_lock, + msg_block, + msg_getblocks, + msg_getdata, + msg_getheaders, + msg_headers, + msg_inv, + msg_sendheaders, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + sync_blocks, + wait_until, +) +DIRECT_FETCH_RESPONSE_TIME = 0.05 -direct_fetch_response_time = 0.05 - -class TestNode(NodeConnCB): +class BaseNode(P2PInterface): def __init__(self): super().__init__() + self.block_announced = False self.last_blockhash_announced = None - def clear_last_announcement(self): - with mininode_lock: - self.block_announced = False - self.last_message.pop("inv", None) - self.last_message.pop("headers", None) - - # Request data for a list of block hashes - def get_data(self, block_hashes): + def send_get_data(self, block_hashes): + """Request data for a list of block hashes.""" msg = msg_getdata() for x in block_hashes: msg.inv.append(CInv(2, x)) - self.connection.send_message(msg) + self.send_message(msg) - def get_headers(self, locator, hashstop): + def send_get_headers(self, locator, hashstop): msg = msg_getheaders() msg.locator.vHave = locator msg.hashstop = hashstop - self.connection.send_message(msg) + self.send_message(msg) def send_block_inv(self, blockhash): msg = msg_inv() msg.inv = [CInv(2, blockhash)] - self.connection.send_message(msg) + self.send_message(msg) + + def send_header_for_blocks(self, new_blocks): + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in new_blocks] + self.send_message(headers_message) - def on_inv(self, conn, message): + def send_getblocks(self, locator): + getblocks_message = msg_getblocks() + getblocks_message.locator.vHave = locator + self.send_message(getblocks_message) + + def wait_for_getdata(self, hash_list, timeout=60): + if hash_list == []: + return + + test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list + wait_until(test_function, timeout=timeout, lock=mininode_lock) + + def wait_for_block_announcement(self, block_hash, timeout=60): + test_function = lambda: self.last_blockhash_announced == block_hash + wait_until(test_function, timeout=timeout, lock=mininode_lock) + + def on_inv(self, message): self.block_announced = True self.last_blockhash_announced = message.inv[-1].hash - def on_headers(self, conn, message): + def on_headers(self, message): if len(message.headers): self.block_announced = True message.headers[-1].calc_sha256() self.last_blockhash_announced = message.headers[-1].sha256 - # Test whether the last announcement we received had the - # right header or the right inv - # inv and headers should be lists of block hashes + def clear_last_announcement(self): + with mininode_lock: + self.block_announced = False + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) + def check_last_announcement(self, headers=None, inv=None): - expect_headers = headers if headers != None else [] - expect_inv = inv if inv != None else [] + """Test whether the last announcement received had the right header or the right inv. + + inv and headers should be lists of block hashes.""" + test_function = lambda: self.block_announced wait_until(test_function, timeout=60, lock=mininode_lock) + with mininode_lock: self.block_announced = False - success = True compare_inv = [] if "inv" in self.last_message: compare_inv = [x.hash for x in self.last_message["inv"].inv] - if compare_inv != expect_inv: - success = False + if inv is not None: + assert_equal(compare_inv, inv) - hash_headers = [] + compare_headers = [] if "headers" in self.last_message: - # treat headers as a list of block hashes - hash_headers = [ x.sha256 for x in self.last_message["headers"].headers ] - if hash_headers != expect_headers: - success = False + compare_headers = [x.sha256 for x in self.last_message["headers"].headers] + if headers is not None: + assert_equal(compare_headers, headers) self.last_message.pop("inv", None) self.last_message.pop("headers", None) - return success - - def wait_for_getdata(self, hash_list, timeout=60): - if hash_list == []: - return - - test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list - wait_until(test_function, timeout=timeout, lock=mininode_lock) - return - - def wait_for_block_announcement(self, block_hash, timeout=60): - test_function = lambda: self.last_blockhash_announced == block_hash - wait_until(test_function, timeout=timeout, lock=mininode_lock) - return - - def send_header_for_blocks(self, new_blocks): - headers_message = msg_headers() - headers_message.headers = [ CBlockHeader(b) for b in new_blocks ] - self.send_message(headers_message) - - def send_getblocks(self, locator): - getblocks_message = msg_getblocks() - getblocks_message.locator.vHave = locator - self.send_message(getblocks_message) class SendHeadersTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - # mine count blocks and return the new tip def mine_blocks(self, count): + """Mine count blocks and return the new tip.""" + # Clear out last block announcement from each p2p listener - [ x.clear_last_announcement() for x in self.p2p_connections ] + [x.clear_last_announcement() for x in self.nodes[0].p2ps] self.nodes[0].generate(count) return int(self.nodes[0].getbestblockhash(), 16) - # mine a reorg that invalidates length blocks (replacing them with - # length+1 blocks). - # Note: we clear the state of our p2p connections after the - # to-be-reorged-out blocks are mined, so that we don't break later tests. - # return the list of block hashes newly mined def mine_reorg(self, length): - self.nodes[0].generate(length) # make sure all invalidated blocks are node0's + """Mine a reorg that invalidates length blocks (replacing them with # length+1 blocks). + + Note: we clear the state of our p2p connections after the + to-be-reorged-out blocks are mined, so that we don't break later tests. + return the list of block hashes newly mined.""" + + self.nodes[0].generate(length) # make sure all invalidated blocks are node0's sync_blocks(self.nodes, wait=0.1) - for x in self.p2p_connections: + for x in self.nodes[0].p2ps: x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16)) x.clear_last_announcement() tip_height = self.nodes[1].getblockcount() - hash_to_invalidate = self.nodes[1].getblockhash(tip_height-(length-1)) + hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1)) self.nodes[1].invalidateblock(hash_to_invalidate) - all_hashes = self.nodes[1].generate(length+1) # Must be longer than the orig chain + all_hashes = self.nodes[1].generate(length + 1) # Must be longer than the orig chain sync_blocks(self.nodes, wait=0.1) return [int(x, 16) for x in all_hashes] def run_test(self): # Setup the p2p connections and start up the network thread. - inv_node = TestNode() - test_node = TestNode() - - self.p2p_connections = [inv_node, test_node] - - connections = [] - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], inv_node)) - # Set nServices to 0 for test_node, so no block download will occur outside of - # direct fetching - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node, services=0)) - inv_node.add_connection(connections[0]) - test_node.add_connection(connections[1]) + inv_node = self.nodes[0].add_p2p_connection(BaseNode()) + # Make sure NODE_NETWORK is not set for test_node, so no block download + # will occur outside of direct fetching + test_node = self.nodes[0].add_p2p_connection(BaseNode(), services=NODE_WITNESS) - NetworkThread().start() # Start up network handling in another thread + network_thread_start() # Test logic begins here inv_node.wait_for_verack() test_node.wait_for_verack() + # Ensure verack's have been processed by our peer + inv_node.sync_with_ping() + test_node.sync_with_ping() + + self.test_null_locators(test_node, inv_node) + self.test_nonnull_locators(test_node, inv_node) + + def test_null_locators(self, test_node, inv_node): + tip = self.nodes[0].getblockheader(self.nodes[0].generate(1)[0]) + tip_hash = int(tip["hash"], 16) + + inv_node.check_last_announcement(inv=[tip_hash], headers=[]) + test_node.check_last_announcement(inv=[tip_hash], headers=[]) + + self.log.info("Verify getheaders with null locator and valid hashstop returns headers.") + test_node.clear_last_announcement() + test_node.send_get_headers(locator=[], hashstop=tip_hash) + test_node.check_last_announcement(headers=[tip_hash]) + + self.log.info("Verify getheaders with null locator and invalid hashstop does not return headers.") + block = create_block(int(tip["hash"], 16), create_coinbase(tip["height"] + 1), tip["mediantime"] + 1) + block.solve() + test_node.send_header_for_blocks([block]) + test_node.clear_last_announcement() + test_node.send_get_headers(locator=[], hashstop=int(block.hash, 16)) + test_node.sync_with_ping() + assert_equal(test_node.block_announced, False) + inv_node.clear_last_announcement() + test_node.send_message(msg_block(block)) + inv_node.check_last_announcement(inv=[int(block.hash, 16)], headers=[]) + + def test_nonnull_locators(self, test_node, inv_node): tip = int(self.nodes[0].getbestblockhash(), 16) # PART 1 @@ -233,30 +284,30 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(4): old_tip = tip tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(inv=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) # Try a few different responses; none should affect next announcement if i == 0: # first request the block - test_node.get_data([tip]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 1: # next try requesting header and block - test_node.get_headers(locator=[old_tip], hashstop=tip) - test_node.get_data([tip]) + test_node.send_get_headers(locator=[old_tip], hashstop=tip) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) - test_node.clear_last_announcement() # since we requested headers... + test_node.clear_last_announcement() # since we requested headers... elif i == 2: # this time announce own block via headers height = self.nodes[0].getblockcount() last_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] block_time = last_time + 1 - new_block = create_block(tip, create_coinbase(height+1), block_time) + new_block = create_block(tip, create_coinbase(height + 1), block_time) new_block.solve() test_node.send_header_for_blocks([new_block]) test_node.wait_for_getdata([new_block.sha256]) test_node.send_message(msg_block(new_block)) - test_node.sync_with_ping() # make sure this block is processed + test_node.sync_with_ping() # make sure this block is processed inv_node.clear_last_announcement() test_node.clear_last_announcement() @@ -267,15 +318,15 @@ class SendHeadersTest(BitcoinTestFramework): # commence and keep working. test_node.send_message(msg_sendheaders()) prev_tip = int(self.nodes[0].getbestblockhash(), 16) - test_node.get_headers(locator=[prev_tip], hashstop=0) + test_node.send_get_headers(locator=[prev_tip], hashstop=0) test_node.sync_with_ping() # Now that we've synced headers, headers announcements should work tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) - height = self.nodes[0].getblockcount()+1 + height = self.nodes[0].getblockcount() + 1 block_time += 10 # Advance far enough ahead for i in range(10): # Mine i blocks, and alternate announcing either via @@ -284,7 +335,7 @@ class SendHeadersTest(BitcoinTestFramework): # with block header, even though the blocks are never requested for j in range(2): blocks = [] - for b in range(i+1): + for b in range(i + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -298,7 +349,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks(blocks) # Test that duplicate inv's won't result in duplicate # getdata requests, or duplicate headers announcements - [ inv_node.send_block_inv(x.sha256) for x in blocks ] + [inv_node.send_block_inv(x.sha256) for x in blocks] test_node.wait_for_getdata([x.sha256 for x in blocks]) inv_node.sync_with_ping() else: @@ -309,7 +360,7 @@ class SendHeadersTest(BitcoinTestFramework): # getdata requests (the check is further down) inv_node.send_header_for_blocks(blocks) inv_node.sync_with_ping() - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() inv_node.sync_with_ping() # This block should not be announced to the inv node (since it also @@ -317,8 +368,8 @@ class SendHeadersTest(BitcoinTestFramework): assert "inv" not in inv_node.last_message assert "headers" not in inv_node.last_message tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) height += 1 block_time += 1 @@ -332,16 +383,16 @@ class SendHeadersTest(BitcoinTestFramework): # First try mining a reorg that can propagate with header announcement new_block_hashes = self.mine_reorg(length=7) tip = new_block_hashes[-1] - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=new_block_hashes), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=new_block_hashes) - block_time += 8 + block_time += 8 # Mine a too-large reorg, which should be announced with a single inv new_block_hashes = self.mine_reorg(length=8) tip = new_block_hashes[-1] - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(inv=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) block_time += 9 @@ -349,42 +400,42 @@ class SendHeadersTest(BitcoinTestFramework): fork_point = int(fork_point, 16) # Use getblocks/getdata - test_node.send_getblocks(locator = [fork_point]) - assert_equal(test_node.check_last_announcement(inv=new_block_hashes), True) - test_node.get_data(new_block_hashes) + test_node.send_getblocks(locator=[fork_point]) + test_node.check_last_announcement(inv=new_block_hashes, headers=[]) + test_node.send_get_data(new_block_hashes) test_node.wait_for_block(new_block_hashes[-1]) for i in range(3): # Mine another block, still should get only an inv tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(inv=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(inv=[tip], headers=[]) if i == 0: # Just get the data -- shouldn't cause headers announcements to resume - test_node.get_data([tip]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 1: # Send a getheaders message that shouldn't trigger headers announcements # to resume (best header sent will be too old) - test_node.get_headers(locator=[fork_point], hashstop=new_block_hashes[1]) - test_node.get_data([tip]) + test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 2: - test_node.get_data([tip]) + test_node.send_get_data([tip]) test_node.wait_for_block(tip) # This time, try sending either a getheaders to trigger resumption - # of headers announcements, or mine a new block and inv it, also + # of headers announcements, or mine a new block and inv it, also # triggering resumption of headers announcements. if j == 0: - test_node.get_headers(locator=[tip], hashstop=0) + test_node.send_get_headers(locator=[tip], hashstop=0) test_node.sync_with_ping() else: test_node.send_block_inv(tip) test_node.sync_with_ping() # New blocks should now be announced with header tip = self.mine_blocks(1) - assert_equal(inv_node.check_last_announcement(inv=[tip]), True) - assert_equal(test_node.check_last_announcement(headers=[tip]), True) + inv_node.check_last_announcement(inv=[tip], headers=[]) + test_node.check_last_announcement(headers=[tip]) self.log.info("Part 3: success!") @@ -404,7 +455,7 @@ class SendHeadersTest(BitcoinTestFramework): height += 1 inv_node.send_message(msg_block(blocks[-1])) - inv_node.sync_with_ping() # Make sure blocks are processed + inv_node.sync_with_ping() # Make sure blocks are processed test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() @@ -423,9 +474,9 @@ class SendHeadersTest(BitcoinTestFramework): test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=direct_fetch_response_time) + test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=DIRECT_FETCH_RESPONSE_TIME) - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() @@ -454,13 +505,13 @@ class SendHeadersTest(BitcoinTestFramework): # both blocks (same work as tip) test_node.send_header_for_blocks(blocks[1:2]) test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=direct_fetch_response_time) + test_node.wait_for_getdata([x.sha256 for x in blocks[0:2]], timeout=DIRECT_FETCH_RESPONSE_TIME) # Announcing 16 more headers should trigger direct fetch for 14 more # blocks test_node.send_header_for_blocks(blocks[2:18]) test_node.sync_with_ping() - test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=direct_fetch_response_time) + test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=DIRECT_FETCH_RESPONSE_TIME) # Announcing 1 more header should not trigger any response test_node.last_message.pop("getdata", None) @@ -472,7 +523,7 @@ class SendHeadersTest(BitcoinTestFramework): self.log.info("Part 4: success!") # Now deliver all those blocks we announced. - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] self.log.info("Part 5: Testing handling of unconnecting headers") # First we test that receipt of an unconnecting header doesn't prevent @@ -494,7 +545,7 @@ class SendHeadersTest(BitcoinTestFramework): test_node.wait_for_getheaders() test_node.send_header_for_blocks(blocks) test_node.wait_for_getdata([x.sha256 for x in blocks]) - [ test_node.send_message(msg_block(x)) for x in blocks ] + [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256) @@ -502,7 +553,7 @@ class SendHeadersTest(BitcoinTestFramework): # Now we test that if we repeatedly don't send connecting headers, we # don't go into an infinite loop trying to get them to connect. MAX_UNCONNECTING_HEADERS = 10 - for j in range(MAX_UNCONNECTING_HEADERS+1): + for j in range(MAX_UNCONNECTING_HEADERS + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) blocks[-1].solve() tip = blocks[-1].sha256 @@ -524,11 +575,11 @@ class SendHeadersTest(BitcoinTestFramework): # Now try to see how many unconnecting headers we can send # before we get disconnected. Should be 5*MAX_UNCONNECTING_HEADERS - for i in range(5*MAX_UNCONNECTING_HEADERS - 1): + for i in range(5 * MAX_UNCONNECTING_HEADERS - 1): # Send a header that doesn't connect, check that we get a getheaders. with mininode_lock: test_node.last_message.pop("getheaders", None) - test_node.send_header_for_blocks([blocks[i%len(blocks)]]) + test_node.send_header_for_blocks([blocks[i % len(blocks)]]) test_node.wait_for_getheaders() # Eventually this stops working. diff --git a/test/functional/signmessages.py b/test/functional/signmessages.py index 52ba6a5ad7..5b6935ceea 100755 --- a/test/functional/signmessages.py +++ b/test/functional/signmessages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC commands for signing and verifying messages.""" @@ -11,6 +11,7 @@ class SignMessagesTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + self.extra_args = [["-addresstype=legacy"]] def run_test(self): message = 'This is just a test message' diff --git a/test/functional/signrawtransactions.py b/test/functional/signrawtransactions.py index 9a45d53cb8..dd0fa6c02c 100755 --- a/test/functional/signrawtransactions.py +++ b/test/functional/signrawtransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction signing using the signrawtransaction RPC.""" diff --git a/test/functional/smartfees.py b/test/functional/smartfees.py index 986f4546a8..68453e50f4 100755 --- a/test/functional/smartfees.py +++ b/test/functional/smartfees.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 2e2db5ffb2..b076c9dd43 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Encode and decode BASE58, P2PKH and P2SH addresses.""" diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index b3671cbdc5..bd3a3b3fab 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -33,24 +33,17 @@ ServiceProxy class: - uses standard Python json lib """ -try: - import http.client as httplib -except ImportError: - import httplib import base64 import decimal +import http.client import json import logging import socket import time -try: - import urllib.parse as urlparse -except ImportError: - import urlparse - -USER_AGENT = "AuthServiceProxy/0.1" +import urllib.parse HTTP_TIMEOUT = 30 +USER_AGENT = "AuthServiceProxy/0.1" log = logging.getLogger("BitcoinRPC") @@ -60,7 +53,7 @@ class JSONRPCException(Exception): errmsg = '%(message)s (%(code)i)' % rpc_error except (KeyError, TypeError): errmsg = '' - Exception.__init__(self, errmsg) + super().__init__(errmsg) self.error = rpc_error @@ -69,28 +62,18 @@ def EncodeDecimal(o): return str(o) raise TypeError(repr(o) + " is not JSON serializable") -class AuthServiceProxy(object): +class AuthServiceProxy(): __id_count = 0 # ensure_ascii: escape unicode as \uXXXX, passed to json.dumps def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connection=None, ensure_ascii=True): self.__service_url = service_url self._service_name = service_name - self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests - self.__url = urlparse.urlparse(service_url) - if self.__url.port is None: - port = 80 - else: - port = self.__url.port - (user, passwd) = (self.__url.username, self.__url.password) - try: - user = user.encode('utf8') - except AttributeError: - pass - try: - passwd = passwd.encode('utf8') - except AttributeError: - pass + self.ensure_ascii = ensure_ascii # can be toggled on the fly by tests + self.__url = urllib.parse.urlparse(service_url) + port = 80 if self.__url.port is None else self.__url.port + user = None if self.__url.username is None else self.__url.username.encode('utf8') + passwd = None if self.__url.password is None else self.__url.password.encode('utf8') authpair = user + b':' + passwd self.__auth_header = b'Basic ' + base64.b64encode(authpair) @@ -98,11 +81,9 @@ class AuthServiceProxy(object): # Callables re-use the connection of the original proxy self.__conn = connection elif self.__url.scheme == 'https': - self.__conn = httplib.HTTPSConnection(self.__url.hostname, port, - timeout=timeout) + self.__conn = http.client.HTTPSConnection(self.__url.hostname, port, timeout=timeout) else: - self.__conn = httplib.HTTPConnection(self.__url.hostname, port, - timeout=timeout) + self.__conn = http.client.HTTPConnection(self.__url.hostname, port, timeout=timeout) def __getattr__(self, name): if name.startswith('__') and name.endswith('__'): @@ -124,31 +105,34 @@ class AuthServiceProxy(object): try: self.__conn.request(method, path, postdata, headers) return self._get_response() - except httplib.BadStatusLine as e: - if e.line == "''": # if connection was closed, try again + except http.client.BadStatusLine as e: + if e.line == "''": # if connection was closed, try again self.__conn.close() self.__conn.request(method, path, postdata, headers) return self._get_response() else: raise - except (BrokenPipeError,ConnectionResetError): + except (BrokenPipeError, ConnectionResetError): # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset # ConnectionResetError happens on FreeBSD with Python 3.4 self.__conn.close() self.__conn.request(method, path, postdata, headers) return self._get_response() - def __call__(self, *args, **argsn): + def get_request(self, *args, **argsn): AuthServiceProxy.__id_count += 1 - log.debug("-%s-> %s %s"%(AuthServiceProxy.__id_count, self._service_name, - json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + log.debug("-%s-> %s %s" % (AuthServiceProxy.__id_count, self._service_name, + json.dumps(args, default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) if args and argsn: raise ValueError('Cannot handle both named and positional arguments') - postdata = json.dumps({'version': '1.1', - 'method': self._service_name, - 'params': args or argsn, - 'id': AuthServiceProxy.__id_count}, default=EncodeDecimal, ensure_ascii=self.ensure_ascii) + return {'version': '1.1', + 'method': self._service_name, + 'params': args or argsn, + 'id': AuthServiceProxy.__id_count} + + def __call__(self, *args, **argsn): + postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) response = self._request('POST', self.__url.path, postdata.encode('utf-8')) if response['error'] is not None: raise JSONRPCException(response['error']) @@ -158,9 +142,9 @@ class AuthServiceProxy(object): else: return response['result'] - def _batch(self, rpc_call_list): + def batch(self, rpc_call_list): postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii) - log.debug("--> "+postdata) + log.debug("--> " + postdata) return self._request('POST', self.__url.path, postdata.encode('utf-8')) def _get_response(self): @@ -187,9 +171,9 @@ class AuthServiceProxy(object): response = json.loads(responsedata, parse_float=decimal.Decimal) elapsed = time.time() - req_start_time if "error" in response and response["error"] is None: - log.debug("<-%s- [%.6f] %s"%(response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) else: - log.debug("<-- [%.6f] %s"%(elapsed,responsedata)) + log.debug("<-- [%.6f] %s" % (elapsed, responsedata)) return response def __truediv__(self, relative_uri): diff --git a/test/functional/test_framework/blockstore.py b/test/functional/test_framework/blockstore.py index 4b2170a03f..6067a407cc 100644 --- a/test/functional/test_framework/blockstore.py +++ b/test/functional/test_framework/blockstore.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """BlockStore and TxStore helper classes.""" @@ -10,7 +10,7 @@ import dbm.dumb as dbmd logger = logging.getLogger("TestFramework.blockstore") -class BlockStore(object): +class BlockStore(): """BlockStore helper class. BlockStore keeps a map of blocks and implements helper functions for @@ -100,7 +100,7 @@ class BlockStore(object): def get_blocks(self, inv): responses = [] for i in inv: - if (i.type == 2): # MSG_BLOCK + if (i.type == 2 or i.type == (2 | (1 << 30))): # MSG_BLOCK or MSG_WITNESS_BLOCK data = self.get(i.hash) if data is not None: # Use msg_generic to avoid re-serialization @@ -127,7 +127,7 @@ class BlockStore(object): locator.vHave = r return locator -class TxStore(object): +class TxStore(): def __init__(self, datadir): self.txDB = dbmd.open(datadir + "/transactions", 'c') @@ -153,7 +153,7 @@ class TxStore(object): def get_transactions(self, inv): responses = [] for i in inv: - if (i.type == 1): # MSG_TX + if (i.type == 1 or i.type == (1 | (1 << 30))): # MSG_TX or MSG_WITNESS_TX tx = self.get(i.hash) if tx is not None: responses.append(msg_generic(b"tx", tx)) diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 55f398719a..642ef98a27 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" diff --git a/test/functional/test_framework/comptool.py b/test/functional/test_framework/comptool.py index bfbc0c3b03..61ea2280e2 100755 --- a/test/functional/test_framework/comptool.py +++ b/test/functional/test_framework/comptool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Compare two or more bitcoinds to each other. @@ -27,7 +27,7 @@ logger=logging.getLogger("TestFramework.comptool") global mininode_lock -class RejectResult(object): +class RejectResult(): """Outcome that expects rejection of a transaction or block.""" def __init__(self, code, reason=b''): self.code = code @@ -39,11 +39,10 @@ class RejectResult(object): def __repr__(self): return '%i:%s' % (self.code,self.reason or '*') -class TestNode(NodeConnCB): +class TestNode(P2PInterface): def __init__(self, block_store, tx_store): super().__init__() - self.conn = None self.bestblockhash = None self.block_store = block_store self.block_request_map = {} @@ -58,43 +57,40 @@ class TestNode(NodeConnCB): self.lastInv = [] self.closed = False - def on_close(self, conn): + def on_close(self): self.closed = True - def add_connection(self, conn): - self.conn = conn - - def on_headers(self, conn, message): + def on_headers(self, message): if len(message.headers) > 0: best_header = message.headers[-1] best_header.calc_sha256() self.bestblockhash = best_header.sha256 - def on_getheaders(self, conn, message): + def on_getheaders(self, message): response = self.block_store.headers_for(message.locator, message.hashstop) if response is not None: - conn.send_message(response) + self.send_message(response) - def on_getdata(self, conn, message): - [conn.send_message(r) for r in self.block_store.get_blocks(message.inv)] - [conn.send_message(r) for r in self.tx_store.get_transactions(message.inv)] + def on_getdata(self, message): + [self.send_message(r) for r in self.block_store.get_blocks(message.inv)] + [self.send_message(r) for r in self.tx_store.get_transactions(message.inv)] for i in message.inv: - if i.type == 1: + if i.type == 1 or i.type == 1 | (1 << 30): # MSG_TX or MSG_WITNESS_TX self.tx_request_map[i.hash] = True - elif i.type == 2: + elif i.type == 2 or i.type == 2 | (1 << 30): # MSG_BLOCK or MSG_WITNESS_BLOCK self.block_request_map[i.hash] = True - def on_inv(self, conn, message): + def on_inv(self, message): self.lastInv = [x.hash for x in message.inv] - def on_pong(self, conn, message): + def on_pong(self, message): try: del self.pingMap[message.nonce] except KeyError: raise AssertionError("Got pong for unknown ping [%s]" % repr(message)) - def on_reject(self, conn, message): + def on_reject(self, message): if message.message == b'tx': self.tx_reject_map[message.data] = RejectResult(message.code, message.reason) if message.message == b'block': @@ -102,30 +98,30 @@ class TestNode(NodeConnCB): def send_inv(self, obj): mtype = 2 if isinstance(obj, CBlock) else 1 - self.conn.send_message(msg_inv([CInv(mtype, obj.sha256)])) + self.send_message(msg_inv([CInv(mtype, obj.sha256)])) def send_getheaders(self): # We ask for headers from their last tip. m = msg_getheaders() m.locator = self.block_store.get_locator(self.bestblockhash) - self.conn.send_message(m) + self.send_message(m) def send_header(self, header): m = msg_headers() m.headers.append(header) - self.conn.send_message(m) + self.send_message(m) # This assumes BIP31 def send_ping(self, nonce): self.pingMap[nonce] = True - self.conn.send_message(msg_ping(nonce)) + self.send_message(msg_ping(nonce)) def received_ping_response(self, nonce): return nonce not in self.pingMap def send_mempool(self): self.lastInv = [] - self.conn.send_message(msg_mempool()) + self.send_message(msg_mempool()) # TestInstance: # @@ -156,18 +152,17 @@ class TestNode(NodeConnCB): # across all connections. (If outcome of final tx is specified as true # or false, then only the last tx is tested against outcome.) -class TestInstance(object): +class TestInstance(): def __init__(self, objects=None, sync_every_block=True, sync_every_tx=False): self.blocks_and_transactions = objects if objects else [] self.sync_every_block = sync_every_block self.sync_every_tx = sync_every_tx -class TestManager(object): +class TestManager(): def __init__(self, testgen, datadir): self.test_generator = testgen - self.connections = [] - self.test_nodes = [] + self.p2p_connections= [] self.block_store = BlockStore(datadir) self.tx_store = TxStore(datadir) self.ping_counter = 1 @@ -175,28 +170,24 @@ class TestManager(object): def add_all_connections(self, nodes): for i in range(len(nodes)): # Create a p2p connection to each node - test_node = TestNode(self.block_store, self.tx_store) - self.test_nodes.append(test_node) - self.connections.append(NodeConn('127.0.0.1', p2p_port(i), nodes[i], test_node)) - # Make sure the TestNode (callback class) has a reference to its - # associated NodeConn - test_node.add_connection(self.connections[-1]) + node = TestNode(self.block_store, self.tx_store) + node.peer_connect('127.0.0.1', p2p_port(i)) + self.p2p_connections.append(node) def clear_all_connections(self): - self.connections = [] - self.test_nodes = [] + self.p2p_connections = [] def wait_for_disconnections(self): def disconnected(): - return all(node.closed for node in self.test_nodes) + return all(node.closed for node in self.p2p_connections) wait_until(disconnected, timeout=10, lock=mininode_lock) def wait_for_verack(self): - return all(node.wait_for_verack() for node in self.test_nodes) + return all(node.wait_for_verack() for node in self.p2p_connections) def wait_for_pings(self, counter): def received_pongs(): - return all(node.received_ping_response(counter) for node in self.test_nodes) + return all(node.received_ping_response(counter) for node in self.p2p_connections) wait_until(received_pongs, lock=mininode_lock) # sync_blocks: Wait for all connections to request the blockhash given @@ -206,17 +197,17 @@ class TestManager(object): def blocks_requested(): return all( blockhash in node.block_request_map and node.block_request_map[blockhash] - for node in self.test_nodes + for node in self.p2p_connections ) # --> error if not requested wait_until(blocks_requested, attempts=20*num_blocks, lock=mininode_lock) # Send getheaders message - [ c.cb.send_getheaders() for c in self.connections ] + [ c.send_getheaders() for c in self.p2p_connections ] # Send ping and wait for response -- synchronization hack - [ c.cb.send_ping(self.ping_counter) for c in self.connections ] + [ c.send_ping(self.ping_counter) for c in self.p2p_connections ] self.wait_for_pings(self.ping_counter) self.ping_counter += 1 @@ -226,42 +217,42 @@ class TestManager(object): def transaction_requested(): return all( txhash in node.tx_request_map and node.tx_request_map[txhash] - for node in self.test_nodes + for node in self.p2p_connections ) # --> error if not requested wait_until(transaction_requested, attempts=20*num_events, lock=mininode_lock) # Get the mempool - [ c.cb.send_mempool() for c in self.connections ] + [ c.send_mempool() for c in self.p2p_connections ] # Send ping and wait for response -- synchronization hack - [ c.cb.send_ping(self.ping_counter) for c in self.connections ] + [ c.send_ping(self.ping_counter) for c in self.p2p_connections ] self.wait_for_pings(self.ping_counter) self.ping_counter += 1 # Sort inv responses from each node with mininode_lock: - [ c.cb.lastInv.sort() for c in self.connections ] + [ c.lastInv.sort() for c in self.p2p_connections ] # Verify that the tip of each connection all agree with each other, and # with the expected outcome (if given) def check_results(self, blockhash, outcome): with mininode_lock: - for c in self.connections: + for c in self.p2p_connections: if outcome is None: - if c.cb.bestblockhash != self.connections[0].cb.bestblockhash: + if c.bestblockhash != self.p2p_connections[0].bestblockhash: return False elif isinstance(outcome, RejectResult): # Check that block was rejected w/ code - if c.cb.bestblockhash == blockhash: + if c.bestblockhash == blockhash: return False - if blockhash not in c.cb.block_reject_map: + if blockhash not in c.block_reject_map: logger.error('Block not in reject map: %064x' % (blockhash)) return False - if not outcome.match(c.cb.block_reject_map[blockhash]): - logger.error('Block rejected with %s instead of expected %s: %064x' % (c.cb.block_reject_map[blockhash], outcome, blockhash)) + if not outcome.match(c.block_reject_map[blockhash]): + logger.error('Block rejected with %s instead of expected %s: %064x' % (c.block_reject_map[blockhash], outcome, blockhash)) return False - elif ((c.cb.bestblockhash == blockhash) != outcome): + elif ((c.bestblockhash == blockhash) != outcome): return False return True @@ -273,21 +264,21 @@ class TestManager(object): # a particular tx's existence in the mempool is the same across all nodes. def check_mempool(self, txhash, outcome): with mininode_lock: - for c in self.connections: + for c in self.p2p_connections: if outcome is None: # Make sure the mempools agree with each other - if c.cb.lastInv != self.connections[0].cb.lastInv: + if c.lastInv != self.p2p_connections[0].lastInv: return False elif isinstance(outcome, RejectResult): # Check that tx was rejected w/ code - if txhash in c.cb.lastInv: + if txhash in c.lastInv: return False - if txhash not in c.cb.tx_reject_map: + if txhash not in c.tx_reject_map: logger.error('Tx not in reject map: %064x' % (txhash)) return False - if not outcome.match(c.cb.tx_reject_map[txhash]): - logger.error('Tx rejected with %s instead of expected %s: %064x' % (c.cb.tx_reject_map[txhash], outcome, txhash)) + if not outcome.match(c.tx_reject_map[txhash]): + logger.error('Tx rejected with %s instead of expected %s: %064x' % (c.tx_reject_map[txhash], outcome, txhash)) return False - elif ((txhash in c.cb.lastInv) != outcome): + elif ((txhash in c.lastInv) != outcome): return False return True @@ -295,8 +286,11 @@ class TestManager(object): # Wait until verack is received self.wait_for_verack() - test_number = 1 - for test_instance in self.test_generator.get_tests(): + test_number = 0 + tests = self.test_generator.get_tests() + for test_instance in tests: + test_number += 1 + logger.info("Running test %d: %s line %s" % (test_number, tests.gi_code.co_filename, tests.gi_frame.f_lineno)) # We use these variables to keep track of the last block # and last transaction in the tests, which are used # if we're not syncing on every block or every tx. @@ -329,25 +323,25 @@ class TestManager(object): first_block_with_hash = False with mininode_lock: self.block_store.add_block(block) - for c in self.connections: - if first_block_with_hash and block.sha256 in c.cb.block_request_map and c.cb.block_request_map[block.sha256] == True: + for c in self.p2p_connections: + if first_block_with_hash and block.sha256 in c.block_request_map and c.block_request_map[block.sha256] == True: # There was a previous request for this block hash # Most likely, we delivered a header for this block # but never had the block to respond to the getdata c.send_message(msg_block(block)) else: - c.cb.block_request_map[block.sha256] = False + c.block_request_map[block.sha256] = False # Either send inv's to each node and sync, or add # to invqueue for later inv'ing. if (test_instance.sync_every_block): # if we expect success, send inv and sync every block # if we expect failure, just push the block and see what happens. if outcome == True: - [ c.cb.send_inv(block) for c in self.connections ] + [ c.send_inv(block) for c in self.p2p_connections ] self.sync_blocks(block.sha256, 1) else: - [ c.send_message(msg_block(block)) for c in self.connections ] - [ c.cb.send_ping(self.ping_counter) for c in self.connections ] + [ c.send_message(msg_block(block)) for c in self.p2p_connections ] + [ c.send_ping(self.ping_counter) for c in self.p2p_connections ] self.wait_for_pings(self.ping_counter) self.ping_counter += 1 if (not self.check_results(tip, outcome)): @@ -357,7 +351,7 @@ class TestManager(object): elif isinstance(b_or_t, CBlockHeader): block_header = b_or_t self.block_store.add_header(block_header) - [ c.cb.send_header(block_header) for c in self.connections ] + [ c.send_header(block_header) for c in self.p2p_connections ] else: # Tx test runner assert(isinstance(b_or_t, CTransaction)) @@ -366,11 +360,11 @@ class TestManager(object): # Add to shared tx store and clear map entry with mininode_lock: self.tx_store.add_transaction(tx) - for c in self.connections: - c.cb.tx_request_map[tx.sha256] = False + for c in self.p2p_connections: + c.tx_request_map[tx.sha256] = False # Again, either inv to all nodes or save for later if (test_instance.sync_every_tx): - [ c.cb.send_inv(tx) for c in self.connections ] + [ c.send_inv(tx) for c in self.p2p_connections ] self.sync_transaction(tx.sha256, 1) if (not self.check_mempool(tx.sha256, outcome)): raise AssertionError("Test failed at test %d" % test_number) @@ -378,29 +372,26 @@ class TestManager(object): invqueue.append(CInv(1, tx.sha256)) # Ensure we're not overflowing the inv queue if len(invqueue) == MAX_INV_SZ: - [ c.send_message(msg_inv(invqueue)) for c in self.connections ] + [ c.send_message(msg_inv(invqueue)) for c in self.p2p_connections ] invqueue = [] # Do final sync if we weren't syncing on every block or every tx. if (not test_instance.sync_every_block and block is not None): if len(invqueue) > 0: - [ c.send_message(msg_inv(invqueue)) for c in self.connections ] + [ c.send_message(msg_inv(invqueue)) for c in self.p2p_connections ] invqueue = [] self.sync_blocks(block.sha256, len(test_instance.blocks_and_transactions)) if (not self.check_results(tip, block_outcome)): raise AssertionError("Block test failed at test %d" % test_number) if (not test_instance.sync_every_tx and tx is not None): if len(invqueue) > 0: - [ c.send_message(msg_inv(invqueue)) for c in self.connections ] + [ c.send_message(msg_inv(invqueue)) for c in self.p2p_connections ] invqueue = [] self.sync_transaction(tx.sha256, len(test_instance.blocks_and_transactions)) if (not self.check_mempool(tx.sha256, tx_outcome)): raise AssertionError("Mempool test failed at test %d" % test_number) - logger.info("Test %d: PASS" % test_number) - test_number += 1 - - [ c.disconnect_node() for c in self.connections ] + [ c.disconnect_node() for c in self.p2p_connections ] self.wait_for_disconnections() self.block_store.close() self.tx_store.close() diff --git a/test/functional/test_framework/coverage.py b/test/functional/test_framework/coverage.py index 227b1a17af..f8761f2bb3 100644 --- a/test/functional/test_framework/coverage.py +++ b/test/functional/test_framework/coverage.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for doing coverage analysis on the RPC interface. @@ -14,7 +14,7 @@ import os REFERENCE_FILENAME = 'rpc_interface.txt' -class AuthServiceProxyWrapper(object): +class AuthServiceProxyWrapper(): """ An object that wraps AuthServiceProxy to record specific RPC calls. @@ -31,10 +31,11 @@ class AuthServiceProxyWrapper(object): self.auth_service_proxy_instance = auth_service_proxy_instance self.coverage_logfile = coverage_logfile - def __getattr__(self, *args, **kwargs): - return_val = self.auth_service_proxy_instance.__getattr__( - *args, **kwargs) - + def __getattr__(self, name): + return_val = getattr(self.auth_service_proxy_instance, name) + if not isinstance(return_val, type(self.auth_service_proxy_instance)): + # If proxy getattr returned an unwrapped value, do the same here. + return return_val return AuthServiceProxyWrapper(return_val, self.coverage_logfile) def __call__(self, *args, **kwargs): @@ -44,20 +45,23 @@ class AuthServiceProxyWrapper(object): """ return_val = self.auth_service_proxy_instance.__call__(*args, **kwargs) + self._log_call() + return return_val + + def _log_call(self): rpc_method = self.auth_service_proxy_instance._service_name if self.coverage_logfile: with open(self.coverage_logfile, 'a+', encoding='utf8') as f: f.write("%s\n" % rpc_method) - return return_val - - @property - def url(self): - return self.auth_service_proxy_instance.url - def __truediv__(self, relative_uri): - return AuthServiceProxyWrapper(self.auth_service_proxy_instance / relative_uri) + return AuthServiceProxyWrapper(self.auth_service_proxy_instance / relative_uri, + self.coverage_logfile) + + def get_request(self, *args, **kwargs): + self._log_call() + return self.auth_service_proxy_instance.get_request(*args, **kwargs) def get_filename(dirname, n_node): """ diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 85a6158a2f..aa91fb5b0d 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -84,7 +84,7 @@ def _check_result(val, func, args): ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p ssl.EC_KEY_new_by_curve_name.errcheck = _check_result -class CECKey(object): +class CECKey(): """Wrapper around OpenSSL's EC_KEY""" POINT_CONVERSION_COMPRESSED = 2 diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py new file mode 100644 index 0000000000..a54a0299c7 --- /dev/null +++ b/test/functional/test_framework/messages.py @@ -0,0 +1,1308 @@ +#!/usr/bin/env python3 +# Copyright (c) 2010 ArtForz -- public domain half-a-node +# Copyright (c) 2012 Jeff Garzik +# Copyright (c) 2010-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Bitcoin test framework primitive and message strcutures + +CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....: + data structures that should map to corresponding structures in + bitcoin/primitives + +msg_block, msg_tx, msg_headers, etc.: + data structures that represent network messages + +ser_*, deser_*: functions that handle serialization/deserialization.""" +from codecs import encode +import copy +import hashlib +from io import BytesIO +import random +import socket +import struct +import time + +from test_framework.siphash import siphash256 +from test_framework.util import hex_str_to_bytes, bytes_to_hex_str + +MIN_VERSION_SUPPORTED = 60001 +MY_VERSION = 70014 # past bip-31 for ping/pong +MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" +MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) + +MAX_INV_SZ = 50000 +MAX_BLOCK_BASE_SIZE = 1000000 + +COIN = 100000000 # 1 btc in satoshis + +NODE_NETWORK = (1 << 0) +# NODE_GETUTXO = (1 << 1) +NODE_BLOOM = (1 << 2) +NODE_WITNESS = (1 << 3) +NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5) +NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7) +NODE_NETWORK_LIMITED = (1 << 10) + +# Serialization/deserialization tools +def sha256(s): + return hashlib.new('sha256', s).digest() + +def ripemd160(s): + return hashlib.new('ripemd160', s).digest() + +def hash256(s): + return sha256(sha256(s)) + +def ser_compact_size(l): + r = b"" + if l < 253: + r = struct.pack("B", l) + elif l < 0x10000: + r = struct.pack("<BH", 253, l) + elif l < 0x100000000: + r = struct.pack("<BI", 254, l) + else: + r = struct.pack("<BQ", 255, l) + return r + +def deser_compact_size(f): + nit = struct.unpack("<B", f.read(1))[0] + if nit == 253: + nit = struct.unpack("<H", f.read(2))[0] + elif nit == 254: + nit = struct.unpack("<I", f.read(4))[0] + elif nit == 255: + nit = struct.unpack("<Q", f.read(8))[0] + return nit + +def deser_string(f): + nit = deser_compact_size(f) + return f.read(nit) + +def ser_string(s): + return ser_compact_size(len(s)) + s + +def deser_uint256(f): + r = 0 + for i in range(8): + t = struct.unpack("<I", f.read(4))[0] + r += t << (i * 32) + return r + + +def ser_uint256(u): + rs = b"" + for i in range(8): + rs += struct.pack("<I", u & 0xFFFFFFFF) + u >>= 32 + return rs + + +def uint256_from_str(s): + r = 0 + t = struct.unpack("<IIIIIIII", s[:32]) + for i in range(8): + r += t[i] << (i * 32) + return r + + +def uint256_from_compact(c): + nbytes = (c >> 24) & 0xFF + v = (c & 0xFFFFFF) << (8 * (nbytes - 3)) + return v + + +def deser_vector(f, c): + nit = deser_compact_size(f) + r = [] + for i in range(nit): + t = c() + t.deserialize(f) + r.append(t) + return r + + +# ser_function_name: Allow for an alternate serialization function on the +# entries in the vector (we use this for serializing the vector of transactions +# for a witness block). +def ser_vector(l, ser_function_name=None): + r = ser_compact_size(len(l)) + for i in l: + if ser_function_name: + r += getattr(i, ser_function_name)() + else: + r += i.serialize() + return r + + +def deser_uint256_vector(f): + nit = deser_compact_size(f) + r = [] + for i in range(nit): + t = deser_uint256(f) + r.append(t) + return r + + +def ser_uint256_vector(l): + r = ser_compact_size(len(l)) + for i in l: + r += ser_uint256(i) + return r + + +def deser_string_vector(f): + nit = deser_compact_size(f) + r = [] + for i in range(nit): + t = deser_string(f) + r.append(t) + return r + + +def ser_string_vector(l): + r = ser_compact_size(len(l)) + for sv in l: + r += ser_string(sv) + return r + + +# Deserialize from a hex string representation (eg from RPC) +def FromHex(obj, hex_string): + obj.deserialize(BytesIO(hex_str_to_bytes(hex_string))) + return obj + +# Convert a binary-serializable object to hex (eg for submission via RPC) +def ToHex(obj): + return bytes_to_hex_str(obj.serialize()) + +# Objects that map to bitcoind objects, which can be serialized/deserialized + +class CAddress(): + def __init__(self): + self.nServices = 1 + self.pchReserved = b"\x00" * 10 + b"\xff" * 2 + self.ip = "0.0.0.0" + self.port = 0 + + def deserialize(self, f): + self.nServices = struct.unpack("<Q", f.read(8))[0] + self.pchReserved = f.read(12) + self.ip = socket.inet_ntoa(f.read(4)) + self.port = struct.unpack(">H", f.read(2))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.nServices) + r += self.pchReserved + r += socket.inet_aton(self.ip) + r += struct.pack(">H", self.port) + return r + + def __repr__(self): + return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices, + self.ip, self.port) + +MSG_WITNESS_FLAG = 1<<30 + +class CInv(): + typemap = { + 0: "Error", + 1: "TX", + 2: "Block", + 1|MSG_WITNESS_FLAG: "WitnessTx", + 2|MSG_WITNESS_FLAG : "WitnessBlock", + 4: "CompactBlock" + } + + def __init__(self, t=0, h=0): + self.type = t + self.hash = h + + def deserialize(self, f): + self.type = struct.unpack("<i", f.read(4))[0] + self.hash = deser_uint256(f) + + def serialize(self): + r = b"" + r += struct.pack("<i", self.type) + r += ser_uint256(self.hash) + return r + + def __repr__(self): + return "CInv(type=%s hash=%064x)" \ + % (self.typemap[self.type], self.hash) + + +class CBlockLocator(): + def __init__(self): + self.nVersion = MY_VERSION + self.vHave = [] + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + self.vHave = deser_uint256_vector(f) + + def serialize(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_uint256_vector(self.vHave) + return r + + def __repr__(self): + return "CBlockLocator(nVersion=%i vHave=%s)" \ + % (self.nVersion, repr(self.vHave)) + + +class COutPoint(): + def __init__(self, hash=0, n=0): + self.hash = hash + self.n = n + + def deserialize(self, f): + self.hash = deser_uint256(f) + self.n = struct.unpack("<I", f.read(4))[0] + + def serialize(self): + r = b"" + r += ser_uint256(self.hash) + r += struct.pack("<I", self.n) + return r + + def __repr__(self): + return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n) + + +class CTxIn(): + def __init__(self, outpoint=None, scriptSig=b"", nSequence=0): + if outpoint is None: + self.prevout = COutPoint() + else: + self.prevout = outpoint + self.scriptSig = scriptSig + self.nSequence = nSequence + + def deserialize(self, f): + self.prevout = COutPoint() + self.prevout.deserialize(f) + self.scriptSig = deser_string(f) + self.nSequence = struct.unpack("<I", f.read(4))[0] + + def serialize(self): + r = b"" + r += self.prevout.serialize() + r += ser_string(self.scriptSig) + r += struct.pack("<I", self.nSequence) + return r + + def __repr__(self): + return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ + % (repr(self.prevout), bytes_to_hex_str(self.scriptSig), + self.nSequence) + + +class CTxOut(): + def __init__(self, nValue=0, scriptPubKey=b""): + self.nValue = nValue + self.scriptPubKey = scriptPubKey + + def deserialize(self, f): + self.nValue = struct.unpack("<q", f.read(8))[0] + self.scriptPubKey = deser_string(f) + + def serialize(self): + r = b"" + r += struct.pack("<q", self.nValue) + r += ser_string(self.scriptPubKey) + return r + + def __repr__(self): + return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ + % (self.nValue // COIN, self.nValue % COIN, + bytes_to_hex_str(self.scriptPubKey)) + + +class CScriptWitness(): + def __init__(self): + # stack is a vector of strings + self.stack = [] + + def __repr__(self): + return "CScriptWitness(%s)" % \ + (",".join([bytes_to_hex_str(x) for x in self.stack])) + + def is_null(self): + if self.stack: + return False + return True + + +class CTxInWitness(): + def __init__(self): + self.scriptWitness = CScriptWitness() + + def deserialize(self, f): + self.scriptWitness.stack = deser_string_vector(f) + + def serialize(self): + return ser_string_vector(self.scriptWitness.stack) + + def __repr__(self): + return repr(self.scriptWitness) + + def is_null(self): + return self.scriptWitness.is_null() + + +class CTxWitness(): + def __init__(self): + self.vtxinwit = [] + + def deserialize(self, f): + for i in range(len(self.vtxinwit)): + self.vtxinwit[i].deserialize(f) + + def serialize(self): + r = b"" + # This is different than the usual vector serialization -- + # we omit the length of the vector, which is required to be + # the same length as the transaction's vin vector. + for x in self.vtxinwit: + r += x.serialize() + return r + + def __repr__(self): + return "CTxWitness(%s)" % \ + (';'.join([repr(x) for x in self.vtxinwit])) + + def is_null(self): + for x in self.vtxinwit: + if not x.is_null(): + return False + return True + + +class CTransaction(): + def __init__(self, tx=None): + if tx is None: + self.nVersion = 1 + self.vin = [] + self.vout = [] + self.wit = CTxWitness() + self.nLockTime = 0 + self.sha256 = None + self.hash = None + else: + self.nVersion = tx.nVersion + self.vin = copy.deepcopy(tx.vin) + self.vout = copy.deepcopy(tx.vout) + self.nLockTime = tx.nLockTime + self.sha256 = tx.sha256 + self.hash = tx.hash + self.wit = copy.deepcopy(tx.wit) + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + self.vin = deser_vector(f, CTxIn) + flags = 0 + if len(self.vin) == 0: + flags = struct.unpack("<B", f.read(1))[0] + # Not sure why flags can't be zero, but this + # matches the implementation in bitcoind + if (flags != 0): + self.vin = deser_vector(f, CTxIn) + self.vout = deser_vector(f, CTxOut) + else: + self.vout = deser_vector(f, CTxOut) + if flags != 0: + self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))] + self.wit.deserialize(f) + self.nLockTime = struct.unpack("<I", f.read(4))[0] + self.sha256 = None + self.hash = None + + def serialize_without_witness(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_vector(self.vin) + r += ser_vector(self.vout) + r += struct.pack("<I", self.nLockTime) + return r + + # Only serialize with witness when explicitly called for + def serialize_with_witness(self): + flags = 0 + if not self.wit.is_null(): + flags |= 1 + r = b"" + r += struct.pack("<i", self.nVersion) + if flags: + dummy = [] + r += ser_vector(dummy) + r += struct.pack("<B", flags) + r += ser_vector(self.vin) + r += ser_vector(self.vout) + if flags & 1: + if (len(self.wit.vtxinwit) != len(self.vin)): + # vtxinwit must have the same length as vin + self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)] + for i in range(len(self.wit.vtxinwit), len(self.vin)): + self.wit.vtxinwit.append(CTxInWitness()) + r += self.wit.serialize() + r += struct.pack("<I", self.nLockTime) + return r + + # Regular serialization is with witness -- must explicitly + # call serialize_without_witness to exclude witness data. + def serialize(self): + return self.serialize_with_witness() + + # Recalculate the txid (transaction hash without witness) + def rehash(self): + self.sha256 = None + self.calc_sha256() + + # We will only cache the serialization without witness in + # self.sha256 and self.hash -- those are expected to be the txid. + def calc_sha256(self, with_witness=False): + if with_witness: + # Don't cache the result, just return it + return uint256_from_str(hash256(self.serialize_with_witness())) + + if self.sha256 is None: + self.sha256 = uint256_from_str(hash256(self.serialize_without_witness())) + self.hash = encode(hash256(self.serialize_without_witness())[::-1], 'hex_codec').decode('ascii') + + def is_valid(self): + self.calc_sha256() + for tout in self.vout: + if tout.nValue < 0 or tout.nValue > 21000000 * COIN: + return False + return True + + def __repr__(self): + return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ + % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) + + +class CBlockHeader(): + def __init__(self, header=None): + if header is None: + self.set_null() + else: + self.nVersion = header.nVersion + self.hashPrevBlock = header.hashPrevBlock + self.hashMerkleRoot = header.hashMerkleRoot + self.nTime = header.nTime + self.nBits = header.nBits + self.nNonce = header.nNonce + self.sha256 = header.sha256 + self.hash = header.hash + self.calc_sha256() + + def set_null(self): + self.nVersion = 1 + self.hashPrevBlock = 0 + self.hashMerkleRoot = 0 + self.nTime = 0 + self.nBits = 0 + self.nNonce = 0 + self.sha256 = None + self.hash = None + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + self.hashPrevBlock = deser_uint256(f) + self.hashMerkleRoot = deser_uint256(f) + self.nTime = struct.unpack("<I", f.read(4))[0] + self.nBits = struct.unpack("<I", f.read(4))[0] + self.nNonce = struct.unpack("<I", f.read(4))[0] + self.sha256 = None + self.hash = None + + def serialize(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_uint256(self.hashPrevBlock) + r += ser_uint256(self.hashMerkleRoot) + r += struct.pack("<I", self.nTime) + r += struct.pack("<I", self.nBits) + r += struct.pack("<I", self.nNonce) + return r + + def calc_sha256(self): + if self.sha256 is None: + r = b"" + r += struct.pack("<i", self.nVersion) + r += ser_uint256(self.hashPrevBlock) + r += ser_uint256(self.hashMerkleRoot) + r += struct.pack("<I", self.nTime) + r += struct.pack("<I", self.nBits) + r += struct.pack("<I", self.nNonce) + self.sha256 = uint256_from_str(hash256(r)) + self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii') + + def rehash(self): + self.sha256 = None + self.calc_sha256() + return self.sha256 + + def __repr__(self): + return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \ + % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, + time.ctime(self.nTime), self.nBits, self.nNonce) + + +class CBlock(CBlockHeader): + def __init__(self, header=None): + super(CBlock, self).__init__(header) + self.vtx = [] + + def deserialize(self, f): + super(CBlock, self).deserialize(f) + self.vtx = deser_vector(f, CTransaction) + + def serialize(self, with_witness=False): + r = b"" + r += super(CBlock, self).serialize() + if with_witness: + r += ser_vector(self.vtx, "serialize_with_witness") + else: + r += ser_vector(self.vtx, "serialize_without_witness") + return r + + # Calculate the merkle root given a vector of transaction hashes + @classmethod + def get_merkle_root(cls, hashes): + while len(hashes) > 1: + newhashes = [] + for i in range(0, len(hashes), 2): + i2 = min(i+1, len(hashes)-1) + newhashes.append(hash256(hashes[i] + hashes[i2])) + hashes = newhashes + return uint256_from_str(hashes[0]) + + def calc_merkle_root(self): + hashes = [] + for tx in self.vtx: + tx.calc_sha256() + hashes.append(ser_uint256(tx.sha256)) + return self.get_merkle_root(hashes) + + def calc_witness_merkle_root(self): + # For witness root purposes, the hash of the + # coinbase, with witness, is defined to be 0...0 + hashes = [ser_uint256(0)] + + for tx in self.vtx[1:]: + # Calculate the hashes with witness data + hashes.append(ser_uint256(tx.calc_sha256(True))) + + return self.get_merkle_root(hashes) + + def is_valid(self): + self.calc_sha256() + target = uint256_from_compact(self.nBits) + if self.sha256 > target: + return False + for tx in self.vtx: + if not tx.is_valid(): + return False + if self.calc_merkle_root() != self.hashMerkleRoot: + return False + return True + + def solve(self): + self.rehash() + target = uint256_from_compact(self.nBits) + while self.sha256 > target: + self.nNonce += 1 + self.rehash() + + def __repr__(self): + return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \ + % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, + time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx)) + + +class PrefilledTransaction(): + def __init__(self, index=0, tx = None): + self.index = index + self.tx = tx + + def deserialize(self, f): + self.index = deser_compact_size(f) + self.tx = CTransaction() + self.tx.deserialize(f) + + def serialize(self, with_witness=True): + r = b"" + r += ser_compact_size(self.index) + if with_witness: + r += self.tx.serialize_with_witness() + else: + r += self.tx.serialize_without_witness() + return r + + def serialize_without_witness(self): + return self.serialize(with_witness=False) + + def serialize_with_witness(self): + return self.serialize(with_witness=True) + + def __repr__(self): + return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) + +# This is what we send on the wire, in a cmpctblock message. +class P2PHeaderAndShortIDs(): + def __init__(self): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids_length = 0 + self.shortids = [] + self.prefilled_txn_length = 0 + self.prefilled_txn = [] + + def deserialize(self, f): + self.header.deserialize(f) + self.nonce = struct.unpack("<Q", f.read(8))[0] + self.shortids_length = deser_compact_size(f) + for i in range(self.shortids_length): + # shortids are defined to be 6 bytes in the spec, so append + # two zero bytes and read it in as an 8-byte number + self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0]) + self.prefilled_txn = deser_vector(f, PrefilledTransaction) + self.prefilled_txn_length = len(self.prefilled_txn) + + # When using version 2 compact blocks, we must serialize with_witness. + def serialize(self, with_witness=False): + r = b"" + r += self.header.serialize() + r += struct.pack("<Q", self.nonce) + r += ser_compact_size(self.shortids_length) + for x in self.shortids: + # We only want the first 6 bytes + r += struct.pack("<Q", x)[0:6] + if with_witness: + r += ser_vector(self.prefilled_txn, "serialize_with_witness") + else: + r += ser_vector(self.prefilled_txn, "serialize_without_witness") + return r + + def __repr__(self): + return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn)) + +# P2P version of the above that will use witness serialization (for compact +# block version 2) +class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs): + def serialize(self): + return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True) + +# Calculate the BIP 152-compact blocks shortid for a given transaction hash +def calculate_shortid(k0, k1, tx_hash): + expected_shortid = siphash256(k0, k1, tx_hash) + expected_shortid &= 0x0000ffffffffffff + return expected_shortid + +# This version gets rid of the array lengths, and reinterprets the differential +# encoding into indices that can be used for lookup. +class HeaderAndShortIDs(): + def __init__(self, p2pheaders_and_shortids = None): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids = [] + self.prefilled_txn = [] + self.use_witness = False + + if p2pheaders_and_shortids != None: + self.header = p2pheaders_and_shortids.header + self.nonce = p2pheaders_and_shortids.nonce + self.shortids = p2pheaders_and_shortids.shortids + last_index = -1 + for x in p2pheaders_and_shortids.prefilled_txn: + self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx)) + last_index = self.prefilled_txn[-1].index + + def to_p2p(self): + if self.use_witness: + ret = P2PHeaderAndShortWitnessIDs() + else: + ret = P2PHeaderAndShortIDs() + ret.header = self.header + ret.nonce = self.nonce + ret.shortids_length = len(self.shortids) + ret.shortids = self.shortids + ret.prefilled_txn_length = len(self.prefilled_txn) + ret.prefilled_txn = [] + last_index = -1 + for x in self.prefilled_txn: + ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx)) + last_index = x.index + return ret + + def get_siphash_keys(self): + header_nonce = self.header.serialize() + header_nonce += struct.pack("<Q", self.nonce) + hash_header_nonce_as_str = sha256(header_nonce) + key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0] + key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0] + return [ key0, key1 ] + + # Version 2 compact blocks use wtxid in shortids (rather than txid) + def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False): + self.header = CBlockHeader(block) + self.nonce = nonce + self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ] + self.shortids = [] + self.use_witness = use_witness + [k0, k1] = self.get_siphash_keys() + for i in range(len(block.vtx)): + if i not in prefill_list: + tx_hash = block.vtx[i].sha256 + if use_witness: + tx_hash = block.vtx[i].calc_sha256(with_witness=True) + self.shortids.append(calculate_shortid(k0, k1, tx_hash)) + + def __repr__(self): + return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn)) + + +class BlockTransactionsRequest(): + + def __init__(self, blockhash=0, indexes = None): + self.blockhash = blockhash + self.indexes = indexes if indexes != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + indexes_length = deser_compact_size(f) + for i in range(indexes_length): + self.indexes.append(deser_compact_size(f)) + + def serialize(self): + r = b"" + r += ser_uint256(self.blockhash) + r += ser_compact_size(len(self.indexes)) + for x in self.indexes: + r += ser_compact_size(x) + return r + + # helper to set the differentially encoded indexes from absolute ones + def from_absolute(self, absolute_indexes): + self.indexes = [] + last_index = -1 + for x in absolute_indexes: + self.indexes.append(x-last_index-1) + last_index = x + + def to_absolute(self): + absolute_indexes = [] + last_index = -1 + for x in self.indexes: + absolute_indexes.append(x+last_index+1) + last_index = absolute_indexes[-1] + return absolute_indexes + + def __repr__(self): + return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes)) + + +class BlockTransactions(): + + def __init__(self, blockhash=0, transactions = None): + self.blockhash = blockhash + self.transactions = transactions if transactions != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + self.transactions = deser_vector(f, CTransaction) + + def serialize(self, with_witness=True): + r = b"" + r += ser_uint256(self.blockhash) + if with_witness: + r += ser_vector(self.transactions, "serialize_with_witness") + else: + r += ser_vector(self.transactions, "serialize_without_witness") + return r + + def __repr__(self): + return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions)) + + +# Objects that correspond to messages on the wire +class msg_version(): + command = b"version" + + def __init__(self): + self.nVersion = MY_VERSION + self.nServices = NODE_NETWORK | NODE_WITNESS + self.nTime = int(time.time()) + self.addrTo = CAddress() + self.addrFrom = CAddress() + self.nNonce = random.getrandbits(64) + self.strSubVer = MY_SUBVERSION + self.nStartingHeight = -1 + self.nRelay = MY_RELAY + + def deserialize(self, f): + self.nVersion = struct.unpack("<i", f.read(4))[0] + if self.nVersion == 10300: + self.nVersion = 300 + self.nServices = struct.unpack("<Q", f.read(8))[0] + self.nTime = struct.unpack("<q", f.read(8))[0] + self.addrTo = CAddress() + self.addrTo.deserialize(f) + + if self.nVersion >= 106: + self.addrFrom = CAddress() + self.addrFrom.deserialize(f) + self.nNonce = struct.unpack("<Q", f.read(8))[0] + self.strSubVer = deser_string(f) + else: + self.addrFrom = None + self.nNonce = None + self.strSubVer = None + self.nStartingHeight = None + + if self.nVersion >= 209: + self.nStartingHeight = struct.unpack("<i", f.read(4))[0] + else: + self.nStartingHeight = None + + if self.nVersion >= 70001: + # Relay field is optional for version 70001 onwards + try: + self.nRelay = struct.unpack("<b", f.read(1))[0] + except: + self.nRelay = 0 + else: + self.nRelay = 0 + + def serialize(self): + r = b"" + r += struct.pack("<i", self.nVersion) + r += struct.pack("<Q", self.nServices) + r += struct.pack("<q", self.nTime) + r += self.addrTo.serialize() + r += self.addrFrom.serialize() + r += struct.pack("<Q", self.nNonce) + r += ser_string(self.strSubVer) + r += struct.pack("<i", self.nStartingHeight) + r += struct.pack("<b", self.nRelay) + return r + + def __repr__(self): + return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \ + % (self.nVersion, self.nServices, time.ctime(self.nTime), + repr(self.addrTo), repr(self.addrFrom), self.nNonce, + self.strSubVer, self.nStartingHeight, self.nRelay) + + +class msg_verack(): + command = b"verack" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_verack()" + + +class msg_addr(): + command = b"addr" + + def __init__(self): + self.addrs = [] + + def deserialize(self, f): + self.addrs = deser_vector(f, CAddress) + + def serialize(self): + return ser_vector(self.addrs) + + def __repr__(self): + return "msg_addr(addrs=%s)" % (repr(self.addrs)) + + +class msg_inv(): + command = b"inv" + + def __init__(self, inv=None): + if inv is None: + self.inv = [] + else: + self.inv = inv + + def deserialize(self, f): + self.inv = deser_vector(f, CInv) + + def serialize(self): + return ser_vector(self.inv) + + def __repr__(self): + return "msg_inv(inv=%s)" % (repr(self.inv)) + + +class msg_getdata(): + command = b"getdata" + + def __init__(self, inv=None): + self.inv = inv if inv != None else [] + + def deserialize(self, f): + self.inv = deser_vector(f, CInv) + + def serialize(self): + return ser_vector(self.inv) + + def __repr__(self): + return "msg_getdata(inv=%s)" % (repr(self.inv)) + + +class msg_getblocks(): + command = b"getblocks" + + def __init__(self): + self.locator = CBlockLocator() + self.hashstop = 0 + + def deserialize(self, f): + self.locator = CBlockLocator() + self.locator.deserialize(f) + self.hashstop = deser_uint256(f) + + def serialize(self): + r = b"" + r += self.locator.serialize() + r += ser_uint256(self.hashstop) + return r + + def __repr__(self): + return "msg_getblocks(locator=%s hashstop=%064x)" \ + % (repr(self.locator), self.hashstop) + + +class msg_tx(): + command = b"tx" + + def __init__(self, tx=CTransaction()): + self.tx = tx + + def deserialize(self, f): + self.tx.deserialize(f) + + def serialize(self): + return self.tx.serialize_without_witness() + + def __repr__(self): + return "msg_tx(tx=%s)" % (repr(self.tx)) + +class msg_witness_tx(msg_tx): + + def serialize(self): + return self.tx.serialize_with_witness() + + +class msg_block(): + command = b"block" + + def __init__(self, block=None): + if block is None: + self.block = CBlock() + else: + self.block = block + + def deserialize(self, f): + self.block.deserialize(f) + + def serialize(self): + return self.block.serialize(with_witness=False) + + def __repr__(self): + return "msg_block(block=%s)" % (repr(self.block)) + +# for cases where a user needs tighter control over what is sent over the wire +# note that the user must supply the name of the command, and the data +class msg_generic(): + def __init__(self, command, data=None): + self.command = command + self.data = data + + def serialize(self): + return self.data + + def __repr__(self): + return "msg_generic()" + +class msg_witness_block(msg_block): + + def serialize(self): + r = self.block.serialize(with_witness=True) + return r + +class msg_getaddr(): + command = b"getaddr" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_getaddr()" + + +class msg_ping(): + command = b"ping" + + def __init__(self, nonce=0): + self.nonce = nonce + + def deserialize(self, f): + self.nonce = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.nonce) + return r + + def __repr__(self): + return "msg_ping(nonce=%08x)" % self.nonce + + +class msg_pong(): + command = b"pong" + + def __init__(self, nonce=0): + self.nonce = nonce + + def deserialize(self, f): + self.nonce = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.nonce) + return r + + def __repr__(self): + return "msg_pong(nonce=%08x)" % self.nonce + + +class msg_mempool(): + command = b"mempool" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_mempool()" + +class msg_sendheaders(): + command = b"sendheaders" + + def __init__(self): + pass + + def deserialize(self, f): + pass + + def serialize(self): + return b"" + + def __repr__(self): + return "msg_sendheaders()" + + +# getheaders message has +# number of entries +# vector of hashes +# hash_stop (hash of last desired block header, 0 to get as many as possible) +class msg_getheaders(): + command = b"getheaders" + + def __init__(self): + self.locator = CBlockLocator() + self.hashstop = 0 + + def deserialize(self, f): + self.locator = CBlockLocator() + self.locator.deserialize(f) + self.hashstop = deser_uint256(f) + + def serialize(self): + r = b"" + r += self.locator.serialize() + r += ser_uint256(self.hashstop) + return r + + def __repr__(self): + return "msg_getheaders(locator=%s, stop=%064x)" \ + % (repr(self.locator), self.hashstop) + + +# headers message has +# <count> <vector of block headers> +class msg_headers(): + command = b"headers" + + def __init__(self, headers=None): + self.headers = headers if headers is not None else [] + + def deserialize(self, f): + # comment in bitcoind indicates these should be deserialized as blocks + blocks = deser_vector(f, CBlock) + for x in blocks: + self.headers.append(CBlockHeader(x)) + + def serialize(self): + blocks = [CBlock(x) for x in self.headers] + return ser_vector(blocks) + + def __repr__(self): + return "msg_headers(headers=%s)" % repr(self.headers) + + +class msg_reject(): + command = b"reject" + REJECT_MALFORMED = 1 + + def __init__(self): + self.message = b"" + self.code = 0 + self.reason = b"" + self.data = 0 + + def deserialize(self, f): + self.message = deser_string(f) + self.code = struct.unpack("<B", f.read(1))[0] + self.reason = deser_string(f) + if (self.code != self.REJECT_MALFORMED and + (self.message == b"block" or self.message == b"tx")): + self.data = deser_uint256(f) + + def serialize(self): + r = ser_string(self.message) + r += struct.pack("<B", self.code) + r += ser_string(self.reason) + if (self.code != self.REJECT_MALFORMED and + (self.message == b"block" or self.message == b"tx")): + r += ser_uint256(self.data) + return r + + def __repr__(self): + return "msg_reject: %s %d %s [%064x]" \ + % (self.message, self.code, self.reason, self.data) + +class msg_feefilter(): + command = b"feefilter" + + def __init__(self, feerate=0): + self.feerate = feerate + + def deserialize(self, f): + self.feerate = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<Q", self.feerate) + return r + + def __repr__(self): + return "msg_feefilter(feerate=%08x)" % self.feerate + +class msg_sendcmpct(): + command = b"sendcmpct" + + def __init__(self): + self.announce = False + self.version = 1 + + def deserialize(self, f): + self.announce = struct.unpack("<?", f.read(1))[0] + self.version = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<?", self.announce) + r += struct.pack("<Q", self.version) + return r + + def __repr__(self): + return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version) + +class msg_cmpctblock(): + command = b"cmpctblock" + + def __init__(self, header_and_shortids = None): + self.header_and_shortids = header_and_shortids + + def deserialize(self, f): + self.header_and_shortids = P2PHeaderAndShortIDs() + self.header_and_shortids.deserialize(f) + + def serialize(self): + r = b"" + r += self.header_and_shortids.serialize() + return r + + def __repr__(self): + return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids) + +class msg_getblocktxn(): + command = b"getblocktxn" + + def __init__(self): + self.block_txn_request = None + + def deserialize(self, f): + self.block_txn_request = BlockTransactionsRequest() + self.block_txn_request.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_txn_request.serialize() + return r + + def __repr__(self): + return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request)) + +class msg_blocktxn(): + command = b"blocktxn" + + def __init__(self): + self.block_transactions = BlockTransactions() + + def deserialize(self, f): + self.block_transactions.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_transactions.serialize(with_witness=False) + return r + + def __repr__(self): + return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions)) + +class msg_witness_blocktxn(msg_blocktxn): + def serialize(self): + r = b"" + r += self.block_transactions.serialize(with_witness=True) + return r diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index d072969d7f..fe14591139 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2016 The Bitcoin Core developers +# Copyright (c) 2010-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin P2P network half-a-node. @@ -9,1462 +9,252 @@ This python code was modified from ArtForz' public domain half-a-node, as found in the mini-node branch of http://github.com/jgarzik/pynode. -NodeConn: an object which manages p2p connectivity to a bitcoin node -NodeConnCB: a base class that describes the interface for receiving - callbacks with network messages from a NodeConn -CBlock, CTransaction, CBlockHeader, CTxIn, CTxOut, etc....: - data structures that should map to corresponding structures in - bitcoin/primitives -msg_block, msg_tx, msg_headers, etc.: - data structures that represent network messages -ser_*, deser_*: functions that handle serialization/deserialization -""" - +P2PConnection: A low-level connection object to a node's P2P interface +P2PInterface: A high-level interface object for communicating to a node over P2P""" import asyncore -from codecs import encode from collections import defaultdict -import copy -import hashlib from io import BytesIO import logging -import random import socket import struct import sys -import time -from threading import RLock, Thread - -from test_framework.siphash import siphash256 -from test_framework.util import hex_str_to_bytes, bytes_to_hex_str, wait_until - -BIP0031_VERSION = 60000 -MY_VERSION = 70014 # past bip-31 for ping/pong -MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" -MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37) - -MAX_INV_SZ = 50000 -MAX_BLOCK_BASE_SIZE = 1000000 +import threading -COIN = 100000000 # 1 btc in satoshis - -NODE_NETWORK = (1 << 0) -# NODE_GETUTXO = (1 << 1) -# NODE_BLOOM = (1 << 2) -NODE_WITNESS = (1 << 3) -NODE_UNSUPPORTED_SERVICE_BIT_5 = (1 << 5) -NODE_UNSUPPORTED_SERVICE_BIT_7 = (1 << 7) +from test_framework.messages import * +from test_framework.util import wait_until logger = logging.getLogger("TestFramework.mininode") -# Keep our own socket map for asyncore, so that we can track disconnects -# ourselves (to workaround an issue with closing an asyncore socket when -# using select) -mininode_socket_map = dict() - -# One lock for synchronizing all data access between the networking thread (see -# NetworkThread below) and the thread running the test logic. For simplicity, -# NodeConn acquires this lock whenever delivering a message to a NodeConnCB, -# and whenever adding anything to the send buffer (in send_message()). This -# lock should be acquired in the thread running the test logic to synchronize -# access to any data shared with the NodeConnCB or NodeConn. -mininode_lock = RLock() - -# Serialization/deserialization tools -def sha256(s): - return hashlib.new('sha256', s).digest() - -def ripemd160(s): - return hashlib.new('ripemd160', s).digest() - -def hash256(s): - return sha256(sha256(s)) - -def ser_compact_size(l): - r = b"" - if l < 253: - r = struct.pack("B", l) - elif l < 0x10000: - r = struct.pack("<BH", 253, l) - elif l < 0x100000000: - r = struct.pack("<BI", 254, l) - else: - r = struct.pack("<BQ", 255, l) - return r - -def deser_compact_size(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] - return nit - -def deser_string(f): - nit = deser_compact_size(f) - return f.read(nit) - -def ser_string(s): - return ser_compact_size(len(s)) + s - -def deser_uint256(f): - r = 0 - for i in range(8): - t = struct.unpack("<I", f.read(4))[0] - r += t << (i * 32) - return r - - -def ser_uint256(u): - rs = b"" - for i in range(8): - rs += struct.pack("<I", u & 0xFFFFFFFF) - u >>= 32 - return rs - - -def uint256_from_str(s): - r = 0 - t = struct.unpack("<IIIIIIII", s[:32]) - for i in range(8): - r += t[i] << (i * 32) - return r - - -def uint256_from_compact(c): - nbytes = (c >> 24) & 0xFF - v = (c & 0xFFFFFF) << (8 * (nbytes - 3)) - return v - - -def deser_vector(f, c): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = c() - t.deserialize(f) - r.append(t) - return r - - -# ser_function_name: Allow for an alternate serialization function on the -# entries in the vector (we use this for serializing the vector of transactions -# for a witness block). -def ser_vector(l, ser_function_name=None): - r = ser_compact_size(len(l)) - for i in l: - if ser_function_name: - r += getattr(i, ser_function_name)() - else: - r += i.serialize() - return r - - -def deser_uint256_vector(f): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = deser_uint256(f) - r.append(t) - return r - - -def ser_uint256_vector(l): - r = ser_compact_size(len(l)) - for i in l: - r += ser_uint256(i) - return r - - -def deser_string_vector(f): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = deser_string(f) - r.append(t) - return r - - -def ser_string_vector(l): - r = ser_compact_size(len(l)) - for sv in l: - r += ser_string(sv) - return r - - -def deser_int_vector(f): - nit = deser_compact_size(f) - r = [] - for i in range(nit): - t = struct.unpack("<i", f.read(4))[0] - r.append(t) - return r - - -def ser_int_vector(l): - r = ser_compact_size(len(l)) - for i in l: - r += struct.pack("<i", i) - return r - -# Deserialize from a hex string representation (eg from RPC) -def FromHex(obj, hex_string): - obj.deserialize(BytesIO(hex_str_to_bytes(hex_string))) - return obj - -# Convert a binary-serializable object to hex (eg for submission via RPC) -def ToHex(obj): - return bytes_to_hex_str(obj.serialize()) - -# Objects that map to bitcoind objects, which can be serialized/deserialized - -class CAddress(object): - def __init__(self): - self.nServices = 1 - self.pchReserved = b"\x00" * 10 + b"\xff" * 2 - self.ip = "0.0.0.0" - self.port = 0 - - def deserialize(self, f): - self.nServices = struct.unpack("<Q", f.read(8))[0] - self.pchReserved = f.read(12) - self.ip = socket.inet_ntoa(f.read(4)) - self.port = struct.unpack(">H", f.read(2))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.nServices) - r += self.pchReserved - r += socket.inet_aton(self.ip) - r += struct.pack(">H", self.port) - return r - - def __repr__(self): - return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices, - self.ip, self.port) - -MSG_WITNESS_FLAG = 1<<30 - -class CInv(object): - typemap = { - 0: "Error", - 1: "TX", - 2: "Block", - 1|MSG_WITNESS_FLAG: "WitnessTx", - 2|MSG_WITNESS_FLAG : "WitnessBlock", - 4: "CompactBlock" - } - - def __init__(self, t=0, h=0): - self.type = t - self.hash = h - - def deserialize(self, f): - self.type = struct.unpack("<i", f.read(4))[0] - self.hash = deser_uint256(f) - - def serialize(self): - r = b"" - r += struct.pack("<i", self.type) - r += ser_uint256(self.hash) - return r - - def __repr__(self): - return "CInv(type=%s hash=%064x)" \ - % (self.typemap[self.type], self.hash) - - -class CBlockLocator(object): - def __init__(self): - self.nVersion = MY_VERSION - self.vHave = [] - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.vHave = deser_uint256_vector(f) - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_uint256_vector(self.vHave) - return r - - def __repr__(self): - return "CBlockLocator(nVersion=%i vHave=%s)" \ - % (self.nVersion, repr(self.vHave)) - - -class COutPoint(object): - def __init__(self, hash=0, n=0): - self.hash = hash - self.n = n - - def deserialize(self, f): - self.hash = deser_uint256(f) - self.n = struct.unpack("<I", f.read(4))[0] - - def serialize(self): - r = b"" - r += ser_uint256(self.hash) - r += struct.pack("<I", self.n) - return r - - def __repr__(self): - return "COutPoint(hash=%064x n=%i)" % (self.hash, self.n) - - -class CTxIn(object): - def __init__(self, outpoint=None, scriptSig=b"", nSequence=0): - if outpoint is None: - self.prevout = COutPoint() - else: - self.prevout = outpoint - self.scriptSig = scriptSig - self.nSequence = nSequence - - def deserialize(self, f): - self.prevout = COutPoint() - self.prevout.deserialize(f) - self.scriptSig = deser_string(f) - self.nSequence = struct.unpack("<I", f.read(4))[0] - - def serialize(self): - r = b"" - r += self.prevout.serialize() - r += ser_string(self.scriptSig) - r += struct.pack("<I", self.nSequence) - return r - - def __repr__(self): - return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ - % (repr(self.prevout), bytes_to_hex_str(self.scriptSig), - self.nSequence) - - -class CTxOut(object): - def __init__(self, nValue=0, scriptPubKey=b""): - self.nValue = nValue - self.scriptPubKey = scriptPubKey - - def deserialize(self, f): - self.nValue = struct.unpack("<q", f.read(8))[0] - self.scriptPubKey = deser_string(f) - - def serialize(self): - r = b"" - r += struct.pack("<q", self.nValue) - r += ser_string(self.scriptPubKey) - return r - - def __repr__(self): - return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ - % (self.nValue // COIN, self.nValue % COIN, - bytes_to_hex_str(self.scriptPubKey)) - - -class CScriptWitness(object): - def __init__(self): - # stack is a vector of strings - self.stack = [] - - def __repr__(self): - return "CScriptWitness(%s)" % \ - (",".join([bytes_to_hex_str(x) for x in self.stack])) - - def is_null(self): - if self.stack: - return False - return True - - -class CTxInWitness(object): - def __init__(self): - self.scriptWitness = CScriptWitness() - - def deserialize(self, f): - self.scriptWitness.stack = deser_string_vector(f) - - def serialize(self): - return ser_string_vector(self.scriptWitness.stack) - - def __repr__(self): - return repr(self.scriptWitness) - - def is_null(self): - return self.scriptWitness.is_null() - - -class CTxWitness(object): - def __init__(self): - self.vtxinwit = [] - - def deserialize(self, f): - for i in range(len(self.vtxinwit)): - self.vtxinwit[i].deserialize(f) - - def serialize(self): - r = b"" - # This is different than the usual vector serialization -- - # we omit the length of the vector, which is required to be - # the same length as the transaction's vin vector. - for x in self.vtxinwit: - r += x.serialize() - return r - - def __repr__(self): - return "CTxWitness(%s)" % \ - (';'.join([repr(x) for x in self.vtxinwit])) - - def is_null(self): - for x in self.vtxinwit: - if not x.is_null(): - return False - return True - - -class CTransaction(object): - def __init__(self, tx=None): - if tx is None: - self.nVersion = 1 - self.vin = [] - self.vout = [] - self.wit = CTxWitness() - self.nLockTime = 0 - self.sha256 = None - self.hash = None - else: - self.nVersion = tx.nVersion - self.vin = copy.deepcopy(tx.vin) - self.vout = copy.deepcopy(tx.vout) - self.nLockTime = tx.nLockTime - self.sha256 = tx.sha256 - self.hash = tx.hash - self.wit = copy.deepcopy(tx.wit) - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.vin = deser_vector(f, CTxIn) - flags = 0 - if len(self.vin) == 0: - flags = struct.unpack("<B", f.read(1))[0] - # Not sure why flags can't be zero, but this - # matches the implementation in bitcoind - if (flags != 0): - self.vin = deser_vector(f, CTxIn) - self.vout = deser_vector(f, CTxOut) - else: - self.vout = deser_vector(f, CTxOut) - if flags != 0: - self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))] - self.wit.deserialize(f) - self.nLockTime = struct.unpack("<I", f.read(4))[0] - self.sha256 = None - self.hash = None - - def serialize_without_witness(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_vector(self.vin) - r += ser_vector(self.vout) - r += struct.pack("<I", self.nLockTime) - return r - - # Only serialize with witness when explicitly called for - def serialize_with_witness(self): - flags = 0 - if not self.wit.is_null(): - flags |= 1 - r = b"" - r += struct.pack("<i", self.nVersion) - if flags: - dummy = [] - r += ser_vector(dummy) - r += struct.pack("<B", flags) - r += ser_vector(self.vin) - r += ser_vector(self.vout) - if flags & 1: - if (len(self.wit.vtxinwit) != len(self.vin)): - # vtxinwit must have the same length as vin - self.wit.vtxinwit = self.wit.vtxinwit[:len(self.vin)] - for i in range(len(self.wit.vtxinwit), len(self.vin)): - self.wit.vtxinwit.append(CTxInWitness()) - r += self.wit.serialize() - r += struct.pack("<I", self.nLockTime) - return r - - # Regular serialization is without witness -- must explicitly - # call serialize_with_witness to include witness data. - def serialize(self): - return self.serialize_without_witness() - - # Recalculate the txid (transaction hash without witness) - def rehash(self): - self.sha256 = None - self.calc_sha256() - - # We will only cache the serialization without witness in - # self.sha256 and self.hash -- those are expected to be the txid. - def calc_sha256(self, with_witness=False): - if with_witness: - # Don't cache the result, just return it - return uint256_from_str(hash256(self.serialize_with_witness())) - - if self.sha256 is None: - self.sha256 = uint256_from_str(hash256(self.serialize_without_witness())) - self.hash = encode(hash256(self.serialize())[::-1], 'hex_codec').decode('ascii') - - def is_valid(self): - self.calc_sha256() - for tout in self.vout: - if tout.nValue < 0 or tout.nValue > 21000000 * COIN: - return False - return True - - def __repr__(self): - return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ - % (self.nVersion, repr(self.vin), repr(self.vout), repr(self.wit), self.nLockTime) - - -class CBlockHeader(object): - def __init__(self, header=None): - if header is None: - self.set_null() - else: - self.nVersion = header.nVersion - self.hashPrevBlock = header.hashPrevBlock - self.hashMerkleRoot = header.hashMerkleRoot - self.nTime = header.nTime - self.nBits = header.nBits - self.nNonce = header.nNonce - self.sha256 = header.sha256 - self.hash = header.hash - self.calc_sha256() - - def set_null(self): - self.nVersion = 1 - self.hashPrevBlock = 0 - self.hashMerkleRoot = 0 - self.nTime = 0 - self.nBits = 0 - self.nNonce = 0 - self.sha256 = None - self.hash = None - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.hashPrevBlock = deser_uint256(f) - self.hashMerkleRoot = deser_uint256(f) - self.nTime = struct.unpack("<I", f.read(4))[0] - self.nBits = struct.unpack("<I", f.read(4))[0] - self.nNonce = struct.unpack("<I", f.read(4))[0] - self.sha256 = None - self.hash = None - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_uint256(self.hashPrevBlock) - r += ser_uint256(self.hashMerkleRoot) - r += struct.pack("<I", self.nTime) - r += struct.pack("<I", self.nBits) - r += struct.pack("<I", self.nNonce) - return r - - def calc_sha256(self): - if self.sha256 is None: - r = b"" - r += struct.pack("<i", self.nVersion) - r += ser_uint256(self.hashPrevBlock) - r += ser_uint256(self.hashMerkleRoot) - r += struct.pack("<I", self.nTime) - r += struct.pack("<I", self.nBits) - r += struct.pack("<I", self.nNonce) - self.sha256 = uint256_from_str(hash256(r)) - self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii') - - def rehash(self): - self.sha256 = None - self.calc_sha256() - return self.sha256 - - def __repr__(self): - return "CBlockHeader(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x)" \ - % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, - time.ctime(self.nTime), self.nBits, self.nNonce) - - -class CBlock(CBlockHeader): - def __init__(self, header=None): - super(CBlock, self).__init__(header) - self.vtx = [] - - def deserialize(self, f): - super(CBlock, self).deserialize(f) - self.vtx = deser_vector(f, CTransaction) - - def serialize(self, with_witness=False): - r = b"" - r += super(CBlock, self).serialize() - if with_witness: - r += ser_vector(self.vtx, "serialize_with_witness") - else: - r += ser_vector(self.vtx) - return r - - # Calculate the merkle root given a vector of transaction hashes - @classmethod - def get_merkle_root(cls, hashes): - while len(hashes) > 1: - newhashes = [] - for i in range(0, len(hashes), 2): - i2 = min(i+1, len(hashes)-1) - newhashes.append(hash256(hashes[i] + hashes[i2])) - hashes = newhashes - return uint256_from_str(hashes[0]) - - def calc_merkle_root(self): - hashes = [] - for tx in self.vtx: - tx.calc_sha256() - hashes.append(ser_uint256(tx.sha256)) - return self.get_merkle_root(hashes) - - def calc_witness_merkle_root(self): - # For witness root purposes, the hash of the - # coinbase, with witness, is defined to be 0...0 - hashes = [ser_uint256(0)] - - for tx in self.vtx[1:]: - # Calculate the hashes with witness data - hashes.append(ser_uint256(tx.calc_sha256(True))) - - return self.get_merkle_root(hashes) - - def is_valid(self): - self.calc_sha256() - target = uint256_from_compact(self.nBits) - if self.sha256 > target: - return False - for tx in self.vtx: - if not tx.is_valid(): - return False - if self.calc_merkle_root() != self.hashMerkleRoot: - return False - return True - - def solve(self): - self.rehash() - target = uint256_from_compact(self.nBits) - while self.sha256 > target: - self.nNonce += 1 - self.rehash() - - def __repr__(self): - return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \ - % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, - time.ctime(self.nTime), self.nBits, self.nNonce, repr(self.vtx)) - - -class CUnsignedAlert(object): - def __init__(self): - self.nVersion = 1 - self.nRelayUntil = 0 - self.nExpiration = 0 - self.nID = 0 - self.nCancel = 0 - self.setCancel = [] - self.nMinVer = 0 - self.nMaxVer = 0 - self.setSubVer = [] - self.nPriority = 0 - self.strComment = b"" - self.strStatusBar = b"" - self.strReserved = b"" - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - self.nRelayUntil = struct.unpack("<q", f.read(8))[0] - self.nExpiration = struct.unpack("<q", f.read(8))[0] - self.nID = struct.unpack("<i", f.read(4))[0] - self.nCancel = struct.unpack("<i", f.read(4))[0] - self.setCancel = deser_int_vector(f) - self.nMinVer = struct.unpack("<i", f.read(4))[0] - self.nMaxVer = struct.unpack("<i", f.read(4))[0] - self.setSubVer = deser_string_vector(f) - self.nPriority = struct.unpack("<i", f.read(4))[0] - self.strComment = deser_string(f) - self.strStatusBar = deser_string(f) - self.strReserved = deser_string(f) - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += struct.pack("<q", self.nRelayUntil) - r += struct.pack("<q", self.nExpiration) - r += struct.pack("<i", self.nID) - r += struct.pack("<i", self.nCancel) - r += ser_int_vector(self.setCancel) - r += struct.pack("<i", self.nMinVer) - r += struct.pack("<i", self.nMaxVer) - r += ser_string_vector(self.setSubVer) - r += struct.pack("<i", self.nPriority) - r += ser_string(self.strComment) - r += ser_string(self.strStatusBar) - r += ser_string(self.strReserved) - return r - - def __repr__(self): - return "CUnsignedAlert(nVersion %d, nRelayUntil %d, nExpiration %d, nID %d, nCancel %d, nMinVer %d, nMaxVer %d, nPriority %d, strComment %s, strStatusBar %s, strReserved %s)" \ - % (self.nVersion, self.nRelayUntil, self.nExpiration, self.nID, - self.nCancel, self.nMinVer, self.nMaxVer, self.nPriority, - self.strComment, self.strStatusBar, self.strReserved) - - -class CAlert(object): - def __init__(self): - self.vchMsg = b"" - self.vchSig = b"" - - def deserialize(self, f): - self.vchMsg = deser_string(f) - self.vchSig = deser_string(f) - - def serialize(self): - r = b"" - r += ser_string(self.vchMsg) - r += ser_string(self.vchSig) - return r - - def __repr__(self): - return "CAlert(vchMsg.sz %d, vchSig.sz %d)" \ - % (len(self.vchMsg), len(self.vchSig)) - - -class PrefilledTransaction(object): - def __init__(self, index=0, tx = None): - self.index = index - self.tx = tx - - def deserialize(self, f): - self.index = deser_compact_size(f) - self.tx = CTransaction() - self.tx.deserialize(f) - - def serialize(self, with_witness=False): - r = b"" - r += ser_compact_size(self.index) - if with_witness: - r += self.tx.serialize_with_witness() - else: - r += self.tx.serialize_without_witness() - return r - - def serialize_with_witness(self): - return self.serialize(with_witness=True) - - def __repr__(self): - return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) - -# This is what we send on the wire, in a cmpctblock message. -class P2PHeaderAndShortIDs(object): - def __init__(self): - self.header = CBlockHeader() - self.nonce = 0 - self.shortids_length = 0 - self.shortids = [] - self.prefilled_txn_length = 0 - self.prefilled_txn = [] - - def deserialize(self, f): - self.header.deserialize(f) - self.nonce = struct.unpack("<Q", f.read(8))[0] - self.shortids_length = deser_compact_size(f) - for i in range(self.shortids_length): - # shortids are defined to be 6 bytes in the spec, so append - # two zero bytes and read it in as an 8-byte number - self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0]) - self.prefilled_txn = deser_vector(f, PrefilledTransaction) - self.prefilled_txn_length = len(self.prefilled_txn) - - # When using version 2 compact blocks, we must serialize with_witness. - def serialize(self, with_witness=False): - r = b"" - r += self.header.serialize() - r += struct.pack("<Q", self.nonce) - r += ser_compact_size(self.shortids_length) - for x in self.shortids: - # We only want the first 6 bytes - r += struct.pack("<Q", x)[0:6] - if with_witness: - r += ser_vector(self.prefilled_txn, "serialize_with_witness") - else: - r += ser_vector(self.prefilled_txn) - return r - - def __repr__(self): - return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn)) - -# P2P version of the above that will use witness serialization (for compact -# block version 2) -class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs): - def serialize(self): - return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True) - -# Calculate the BIP 152-compact blocks shortid for a given transaction hash -def calculate_shortid(k0, k1, tx_hash): - expected_shortid = siphash256(k0, k1, tx_hash) - expected_shortid &= 0x0000ffffffffffff - return expected_shortid - -# This version gets rid of the array lengths, and reinterprets the differential -# encoding into indices that can be used for lookup. -class HeaderAndShortIDs(object): - def __init__(self, p2pheaders_and_shortids = None): - self.header = CBlockHeader() - self.nonce = 0 - self.shortids = [] - self.prefilled_txn = [] - self.use_witness = False - - if p2pheaders_and_shortids != None: - self.header = p2pheaders_and_shortids.header - self.nonce = p2pheaders_and_shortids.nonce - self.shortids = p2pheaders_and_shortids.shortids - last_index = -1 - for x in p2pheaders_and_shortids.prefilled_txn: - self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx)) - last_index = self.prefilled_txn[-1].index - - def to_p2p(self): - if self.use_witness: - ret = P2PHeaderAndShortWitnessIDs() - else: - ret = P2PHeaderAndShortIDs() - ret.header = self.header - ret.nonce = self.nonce - ret.shortids_length = len(self.shortids) - ret.shortids = self.shortids - ret.prefilled_txn_length = len(self.prefilled_txn) - ret.prefilled_txn = [] - last_index = -1 - for x in self.prefilled_txn: - ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx)) - last_index = x.index - return ret - - def get_siphash_keys(self): - header_nonce = self.header.serialize() - header_nonce += struct.pack("<Q", self.nonce) - hash_header_nonce_as_str = sha256(header_nonce) - key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0] - key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0] - return [ key0, key1 ] - - # Version 2 compact blocks use wtxid in shortids (rather than txid) - def initialize_from_block(self, block, nonce=0, prefill_list = [0], use_witness = False): - self.header = CBlockHeader(block) - self.nonce = nonce - self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ] - self.shortids = [] - self.use_witness = use_witness - [k0, k1] = self.get_siphash_keys() - for i in range(len(block.vtx)): - if i not in prefill_list: - tx_hash = block.vtx[i].sha256 - if use_witness: - tx_hash = block.vtx[i].calc_sha256(with_witness=True) - self.shortids.append(calculate_shortid(k0, k1, tx_hash)) - - def __repr__(self): - return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn)) - - -class BlockTransactionsRequest(object): - - def __init__(self, blockhash=0, indexes = None): - self.blockhash = blockhash - self.indexes = indexes if indexes != None else [] - - def deserialize(self, f): - self.blockhash = deser_uint256(f) - indexes_length = deser_compact_size(f) - for i in range(indexes_length): - self.indexes.append(deser_compact_size(f)) - - def serialize(self): - r = b"" - r += ser_uint256(self.blockhash) - r += ser_compact_size(len(self.indexes)) - for x in self.indexes: - r += ser_compact_size(x) - return r - - # helper to set the differentially encoded indexes from absolute ones - def from_absolute(self, absolute_indexes): - self.indexes = [] - last_index = -1 - for x in absolute_indexes: - self.indexes.append(x-last_index-1) - last_index = x - - def to_absolute(self): - absolute_indexes = [] - last_index = -1 - for x in self.indexes: - absolute_indexes.append(x+last_index+1) - last_index = absolute_indexes[-1] - return absolute_indexes - - def __repr__(self): - return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes)) - - -class BlockTransactions(object): - - def __init__(self, blockhash=0, transactions = None): - self.blockhash = blockhash - self.transactions = transactions if transactions != None else [] - - def deserialize(self, f): - self.blockhash = deser_uint256(f) - self.transactions = deser_vector(f, CTransaction) - - def serialize(self, with_witness=False): - r = b"" - r += ser_uint256(self.blockhash) - if with_witness: - r += ser_vector(self.transactions, "serialize_with_witness") - else: - r += ser_vector(self.transactions) - return r - - def __repr__(self): - return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions)) - - -# Objects that correspond to messages on the wire -class msg_version(object): - command = b"version" - - def __init__(self): - self.nVersion = MY_VERSION - self.nServices = 1 - self.nTime = int(time.time()) - self.addrTo = CAddress() - self.addrFrom = CAddress() - self.nNonce = random.getrandbits(64) - self.strSubVer = MY_SUBVERSION - self.nStartingHeight = -1 - self.nRelay = MY_RELAY - - def deserialize(self, f): - self.nVersion = struct.unpack("<i", f.read(4))[0] - if self.nVersion == 10300: - self.nVersion = 300 - self.nServices = struct.unpack("<Q", f.read(8))[0] - self.nTime = struct.unpack("<q", f.read(8))[0] - self.addrTo = CAddress() - self.addrTo.deserialize(f) - - if self.nVersion >= 106: - self.addrFrom = CAddress() - self.addrFrom.deserialize(f) - self.nNonce = struct.unpack("<Q", f.read(8))[0] - self.strSubVer = deser_string(f) - else: - self.addrFrom = None - self.nNonce = None - self.strSubVer = None - self.nStartingHeight = None - - if self.nVersion >= 209: - self.nStartingHeight = struct.unpack("<i", f.read(4))[0] - else: - self.nStartingHeight = None - - if self.nVersion >= 70001: - # Relay field is optional for version 70001 onwards - try: - self.nRelay = struct.unpack("<b", f.read(1))[0] - except: - self.nRelay = 0 - else: - self.nRelay = 0 - - def serialize(self): - r = b"" - r += struct.pack("<i", self.nVersion) - r += struct.pack("<Q", self.nServices) - r += struct.pack("<q", self.nTime) - r += self.addrTo.serialize() - r += self.addrFrom.serialize() - r += struct.pack("<Q", self.nNonce) - r += ser_string(self.strSubVer) - r += struct.pack("<i", self.nStartingHeight) - r += struct.pack("<b", self.nRelay) - return r - - def __repr__(self): - return 'msg_version(nVersion=%i nServices=%i nTime=%s addrTo=%s addrFrom=%s nNonce=0x%016X strSubVer=%s nStartingHeight=%i nRelay=%i)' \ - % (self.nVersion, self.nServices, time.ctime(self.nTime), - repr(self.addrTo), repr(self.addrFrom), self.nNonce, - self.strSubVer, self.nStartingHeight, self.nRelay) - - -class msg_verack(object): - command = b"verack" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_verack()" - - -class msg_addr(object): - command = b"addr" - - def __init__(self): - self.addrs = [] - - def deserialize(self, f): - self.addrs = deser_vector(f, CAddress) - - def serialize(self): - return ser_vector(self.addrs) - - def __repr__(self): - return "msg_addr(addrs=%s)" % (repr(self.addrs)) - - -class msg_alert(object): - command = b"alert" - - def __init__(self): - self.alert = CAlert() - - def deserialize(self, f): - self.alert = CAlert() - self.alert.deserialize(f) - - def serialize(self): - r = b"" - r += self.alert.serialize() - return r - - def __repr__(self): - return "msg_alert(alert=%s)" % (repr(self.alert), ) - - -class msg_inv(object): - command = b"inv" - - def __init__(self, inv=None): - if inv is None: - self.inv = [] - else: - self.inv = inv - - def deserialize(self, f): - self.inv = deser_vector(f, CInv) - - def serialize(self): - return ser_vector(self.inv) - - def __repr__(self): - return "msg_inv(inv=%s)" % (repr(self.inv)) - - -class msg_getdata(object): - command = b"getdata" - - def __init__(self, inv=None): - self.inv = inv if inv != None else [] - - def deserialize(self, f): - self.inv = deser_vector(f, CInv) - - def serialize(self): - return ser_vector(self.inv) - - def __repr__(self): - return "msg_getdata(inv=%s)" % (repr(self.inv)) - - -class msg_getblocks(object): - command = b"getblocks" - - def __init__(self): - self.locator = CBlockLocator() - self.hashstop = 0 - - def deserialize(self, f): - self.locator = CBlockLocator() - self.locator.deserialize(f) - self.hashstop = deser_uint256(f) - - def serialize(self): - r = b"" - r += self.locator.serialize() - r += ser_uint256(self.hashstop) - return r - - def __repr__(self): - return "msg_getblocks(locator=%s hashstop=%064x)" \ - % (repr(self.locator), self.hashstop) - - -class msg_tx(object): - command = b"tx" - - def __init__(self, tx=CTransaction()): - self.tx = tx - - def deserialize(self, f): - self.tx.deserialize(f) - - def serialize(self): - return self.tx.serialize_without_witness() - - def __repr__(self): - return "msg_tx(tx=%s)" % (repr(self.tx)) - -class msg_witness_tx(msg_tx): - - def serialize(self): - return self.tx.serialize_with_witness() - - -class msg_block(object): - command = b"block" - - def __init__(self, block=None): - if block is None: - self.block = CBlock() - else: - self.block = block - - def deserialize(self, f): - self.block.deserialize(f) - - def serialize(self): - return self.block.serialize() - - def __repr__(self): - return "msg_block(block=%s)" % (repr(self.block)) - -# for cases where a user needs tighter control over what is sent over the wire -# note that the user must supply the name of the command, and the data -class msg_generic(object): - def __init__(self, command, data=None): - self.command = command - self.data = data - - def serialize(self): - return self.data - - def __repr__(self): - return "msg_generic()" - -class msg_witness_block(msg_block): - - def serialize(self): - r = self.block.serialize(with_witness=True) - return r - -class msg_getaddr(object): - command = b"getaddr" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_getaddr()" - - -class msg_ping_prebip31(object): - command = b"ping" +MESSAGEMAP = { + b"addr": msg_addr, + b"block": msg_block, + b"blocktxn": msg_blocktxn, + b"cmpctblock": msg_cmpctblock, + b"feefilter": msg_feefilter, + b"getaddr": msg_getaddr, + b"getblocks": msg_getblocks, + b"getblocktxn": msg_getblocktxn, + b"getdata": msg_getdata, + b"getheaders": msg_getheaders, + b"headers": msg_headers, + b"inv": msg_inv, + b"mempool": msg_mempool, + b"ping": msg_ping, + b"pong": msg_pong, + b"reject": msg_reject, + b"sendcmpct": msg_sendcmpct, + b"sendheaders": msg_sendheaders, + b"tx": msg_tx, + b"verack": msg_verack, + b"version": msg_version, +} + +MAGIC_BYTES = { + "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet + "testnet3": b"\x0b\x11\x09\x07", # testnet3 + "regtest": b"\xfa\xbf\xb5\xda", # regtest +} + +class P2PConnection(asyncore.dispatcher): + """A low-level connection object to a node's P2P interface. + + This class is responsible for: + + - opening and closing the TCP connection to the node + - reading bytes from and writing bytes to the socket + - deserializing and serializing the P2P message header + - logging messages as they are sent and received + + This class contains no logic for handing the P2P message payloads. It must be + sub-classed and the on_message() callback overridden.""" def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_ping() (pre-bip31)" - - -class msg_ping(object): - command = b"ping" - - def __init__(self, nonce=0): - self.nonce = nonce - - def deserialize(self, f): - self.nonce = struct.unpack("<Q", f.read(8))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.nonce) - return r - - def __repr__(self): - return "msg_ping(nonce=%08x)" % self.nonce - - -class msg_pong(object): - command = b"pong" - - def __init__(self, nonce=0): - self.nonce = nonce - - def deserialize(self, f): - self.nonce = struct.unpack("<Q", f.read(8))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.nonce) - return r - - def __repr__(self): - return "msg_pong(nonce=%08x)" % self.nonce - - -class msg_mempool(object): - command = b"mempool" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_mempool()" - -class msg_sendheaders(object): - command = b"sendheaders" - - def __init__(self): - pass - - def deserialize(self, f): - pass - - def serialize(self): - return b"" - - def __repr__(self): - return "msg_sendheaders()" - - -# getheaders message has -# number of entries -# vector of hashes -# hash_stop (hash of last desired block header, 0 to get as many as possible) -class msg_getheaders(object): - command = b"getheaders" - - def __init__(self): - self.locator = CBlockLocator() - self.hashstop = 0 - - def deserialize(self, f): - self.locator = CBlockLocator() - self.locator.deserialize(f) - self.hashstop = deser_uint256(f) - - def serialize(self): - r = b"" - r += self.locator.serialize() - r += ser_uint256(self.hashstop) - return r - - def __repr__(self): - return "msg_getheaders(locator=%s, stop=%064x)" \ - % (repr(self.locator), self.hashstop) + # All P2PConnections must be created before starting the NetworkThread. + # assert that the network thread is not running. + assert not network_thread_running() + super().__init__(map=mininode_socket_map) -# headers message has -# <count> <vector of block headers> -class msg_headers(object): - command = b"headers" - - def __init__(self): - self.headers = [] - - def deserialize(self, f): - # comment in bitcoind indicates these should be deserialized as blocks - blocks = deser_vector(f, CBlock) - for x in blocks: - self.headers.append(CBlockHeader(x)) - - def serialize(self): - blocks = [CBlock(x) for x in self.headers] - return ser_vector(blocks) - - def __repr__(self): - return "msg_headers(headers=%s)" % repr(self.headers) + def peer_connect(self, dstaddr, dstport, net="regtest"): + self.dstaddr = dstaddr + self.dstport = dstport + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.sendbuf = b"" + self.recvbuf = b"" + self.state = "connecting" + self.network = net + self.disconnect = False + logger.info('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) -class msg_reject(object): - command = b"reject" - REJECT_MALFORMED = 1 + try: + self.connect((dstaddr, dstport)) + except: + self.handle_close() - def __init__(self): - self.message = b"" - self.code = 0 - self.reason = b"" - self.data = 0 - - def deserialize(self, f): - self.message = deser_string(f) - self.code = struct.unpack("<B", f.read(1))[0] - self.reason = deser_string(f) - if (self.code != self.REJECT_MALFORMED and - (self.message == b"block" or self.message == b"tx")): - self.data = deser_uint256(f) - - def serialize(self): - r = ser_string(self.message) - r += struct.pack("<B", self.code) - r += ser_string(self.reason) - if (self.code != self.REJECT_MALFORMED and - (self.message == b"block" or self.message == b"tx")): - r += ser_uint256(self.data) - return r - - def __repr__(self): - return "msg_reject: %s %d %s [%064x]" \ - % (self.message, self.code, self.reason, self.data) - -class msg_feefilter(object): - command = b"feefilter" - - def __init__(self, feerate=0): - self.feerate = feerate - - def deserialize(self, f): - self.feerate = struct.unpack("<Q", f.read(8))[0] - - def serialize(self): - r = b"" - r += struct.pack("<Q", self.feerate) - return r - - def __repr__(self): - return "msg_feefilter(feerate=%08x)" % self.feerate - -class msg_sendcmpct(object): - command = b"sendcmpct" + def peer_disconnect(self): + # Connection could have already been closed by other end. + if self.state == "connected": + self.disconnect_node() - def __init__(self): - self.announce = False - self.version = 1 + # Connection and disconnection methods - def deserialize(self, f): - self.announce = struct.unpack("<?", f.read(1))[0] - self.version = struct.unpack("<Q", f.read(8))[0] + def handle_connect(self): + """asyncore callback when a connection is opened.""" + if self.state != "connected": + logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport)) + self.state = "connected" + self.on_open() - def serialize(self): - r = b"" - r += struct.pack("<?", self.announce) - r += struct.pack("<Q", self.version) - return r + def handle_close(self): + """asyncore callback when a connection is closed.""" + logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport)) + self.state = "closed" + self.recvbuf = b"" + self.sendbuf = b"" + try: + self.close() + except: + pass + self.on_close() - def __repr__(self): - return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version) + def disconnect_node(self): + """Disconnect the p2p connection. -class msg_cmpctblock(object): - command = b"cmpctblock" + Called by the test logic thread. Causes the p2p connection + to be disconnected on the next iteration of the asyncore loop.""" + self.disconnect = True - def __init__(self, header_and_shortids = None): - self.header_and_shortids = header_and_shortids + # Socket read methods - def deserialize(self, f): - self.header_and_shortids = P2PHeaderAndShortIDs() - self.header_and_shortids.deserialize(f) + def handle_read(self): + """asyncore callback when data is read from the socket.""" + t = self.recv(8192) + if len(t) > 0: + self.recvbuf += t + self._on_data() - def serialize(self): - r = b"" - r += self.header_and_shortids.serialize() - return r + def _on_data(self): + """Try to read P2P messages from the recv buffer. - def __repr__(self): - return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids) + This method reads data from the buffer in a loop. It deserializes, + parses and verifies the P2P header, then passes the P2P payload to + the on_message callback for processing.""" + try: + while True: + if len(self.recvbuf) < 4: + return + if self.recvbuf[:4] != MAGIC_BYTES[self.network]: + raise ValueError("got garbage %s" % repr(self.recvbuf)) + if len(self.recvbuf) < 4 + 12 + 4 + 4: + return + command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] + msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] + checksum = self.recvbuf[4+12+4:4+12+4+4] + if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen: + return + msg = self.recvbuf[4+12+4+4:4+12+4+4+msglen] + th = sha256(msg) + h = sha256(th) + if checksum != h[:4]: + raise ValueError("got bad checksum " + repr(self.recvbuf)) + self.recvbuf = self.recvbuf[4+12+4+4+msglen:] + if command not in MESSAGEMAP: + raise ValueError("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) + f = BytesIO(msg) + t = MESSAGEMAP[command]() + t.deserialize(f) + self._log_message("receive", t) + self.on_message(t) + except Exception as e: + logger.exception('Error reading message:', repr(e)) + raise -class msg_getblocktxn(object): - command = b"getblocktxn" + def on_message(self, message): + """Callback for processing a P2P payload. Must be overridden by derived class.""" + raise NotImplementedError - def __init__(self): - self.block_txn_request = None + # Socket write methods - def deserialize(self, f): - self.block_txn_request = BlockTransactionsRequest() - self.block_txn_request.deserialize(f) + def writable(self): + """asyncore method to determine whether the handle_write() callback should be called on the next loop.""" + with mininode_lock: + pre_connection = self.state == "connecting" + length = len(self.sendbuf) + return (length > 0 or pre_connection) - def serialize(self): - r = b"" - r += self.block_txn_request.serialize() - return r + def handle_write(self): + """asyncore callback when data should be written to the socket.""" + with mininode_lock: + # asyncore does not expose socket connection, only the first read/write + # event, thus we must check connection manually here to know when we + # actually connect + if self.state == "connecting": + self.handle_connect() + if not self.writable(): + return - def __repr__(self): - return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request)) + try: + sent = self.send(self.sendbuf) + except: + self.handle_close() + return + self.sendbuf = self.sendbuf[sent:] -class msg_blocktxn(object): - command = b"blocktxn" + def send_message(self, message, pushbuf=False): + """Send a P2P message over the socket. - def __init__(self): - self.block_transactions = BlockTransactions() + This method takes a P2P payload, builds the P2P header and adds + the message to the send buffer to be sent over the socket.""" + if self.state != "connected" and not pushbuf: + raise IOError('Not connected, no pushbuf') + self._log_message("send", message) + command = message.command + data = message.serialize() + tmsg = MAGIC_BYTES[self.network] + tmsg += command + tmsg += b"\x00" * (12 - len(command)) + tmsg += struct.pack("<I", len(data)) + th = sha256(data) + h = sha256(th) + tmsg += h[:4] + tmsg += data + with mininode_lock: + if (len(self.sendbuf) == 0 and not pushbuf): + try: + sent = self.send(tmsg) + self.sendbuf = tmsg[sent:] + except BlockingIOError: + self.sendbuf = tmsg + else: + self.sendbuf += tmsg - def deserialize(self, f): - self.block_transactions.deserialize(f) + # Class utility methods - def serialize(self): - r = b"" - r += self.block_transactions.serialize() - return r + def _log_message(self, direction, msg): + """Logs a message being sent or received over the connection.""" + if direction == "send": + log_message = "Send message to " + elif direction == "receive": + log_message = "Received message from " + log_message += "%s:%d: %s" % (self.dstaddr, self.dstport, repr(msg)[:500]) + if len(log_message) > 500: + log_message += "... (msg truncated)" + logger.debug(log_message) - def __repr__(self): - return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions)) -class msg_witness_blocktxn(msg_blocktxn): - def serialize(self): - r = b"" - r += self.block_transactions.serialize(with_witness=True) - return r +class P2PInterface(P2PConnection): + """A high-level P2P interface class for communicating with a Bitcoin node. -class NodeConnCB(object): - """Callback and helper functions for P2P connection to a bitcoind node. + This class provides high-level callbacks for processing P2P message + payloads, as well as convenience methods for interacting with the + node over P2P. Individual testcases should subclass this and override the on_* methods - if they want to alter message handling behaviour. - """ - + if they want to alter message handling behaviour.""" def __init__(self): - # Track whether we have a P2P connection open to the node - self.connected = False - self.connection = None + super().__init__() # Track number of messages of each type received and the most recent # message of each type @@ -1474,100 +264,89 @@ class NodeConnCB(object): # A count of the number of ping messages we've sent to the node self.ping_counter = 1 - # deliver_sleep_time is helpful for debugging race conditions in p2p - # tests; it causes message delivery to sleep for the specified time - # before acquiring the global lock and delivering the next message. - self.deliver_sleep_time = None + # The network services received from the peer + self.nServices = 0 + + def peer_connect(self, *args, services=NODE_NETWORK|NODE_WITNESS, send_version=True, **kwargs): + super().peer_connect(*args, **kwargs) + + if send_version: + # Send a version msg + vt = msg_version() + vt.nServices = services + vt.addrTo.ip = self.dstaddr + vt.addrTo.port = self.dstport + vt.addrFrom.ip = "0.0.0.0" + vt.addrFrom.port = 0 + self.send_message(vt, True) # Message receiving methods - def deliver(self, conn, message): + def on_message(self, message): """Receive message and dispatch message to appropriate callback. We keep a count of how many of each message type has been received - and the most recent message of each type. - - Optionally waits for deliver_sleep_time before dispatching message. - """ - - deliver_sleep = self.get_deliver_sleep_time() - if deliver_sleep is not None: - time.sleep(deliver_sleep) + and the most recent message of each type.""" with mininode_lock: try: command = message.command.decode('ascii') self.message_count[command] += 1 self.last_message[command] = message - getattr(self, 'on_' + command)(conn, message) + getattr(self, 'on_' + command)(message) except: - print("ERROR delivering %s (%s)" % (repr(message), - sys.exc_info()[0])) + print("ERROR delivering %s (%s)" % (repr(message), sys.exc_info()[0])) raise - def get_deliver_sleep_time(self): - with mininode_lock: - return self.deliver_sleep_time - # Callback methods. Can be overridden by subclasses in individual test # cases to provide custom message handling behaviour. - def on_open(self, conn): - self.connected = True - - def on_close(self, conn): - self.connected = False - self.connection = None - - def on_addr(self, conn, message): pass - def on_alert(self, conn, message): pass - def on_block(self, conn, message): pass - def on_blocktxn(self, conn, message): pass - def on_cmpctblock(self, conn, message): pass - def on_feefilter(self, conn, message): pass - def on_getaddr(self, conn, message): pass - def on_getblocks(self, conn, message): pass - def on_getblocktxn(self, conn, message): pass - def on_getdata(self, conn, message): pass - def on_getheaders(self, conn, message): pass - def on_headers(self, conn, message): pass - def on_mempool(self, conn): pass - def on_pong(self, conn, message): pass - def on_reject(self, conn, message): pass - def on_sendcmpct(self, conn, message): pass - def on_sendheaders(self, conn, message): pass - def on_tx(self, conn, message): pass - - def on_inv(self, conn, message): + def on_open(self): + pass + + def on_close(self): + pass + + def on_addr(self, message): pass + def on_block(self, message): pass + def on_blocktxn(self, message): pass + def on_cmpctblock(self, message): pass + def on_feefilter(self, message): pass + def on_getaddr(self, message): pass + def on_getblocks(self, message): pass + def on_getblocktxn(self, message): pass + def on_getdata(self, message): pass + def on_getheaders(self, message): pass + def on_headers(self, message): pass + def on_mempool(self, message): pass + def on_pong(self, message): pass + def on_reject(self, message): pass + def on_sendcmpct(self, message): pass + def on_sendheaders(self, message): pass + def on_tx(self, message): pass + + def on_inv(self, message): want = msg_getdata() for i in message.inv: if i.type != 0: want.inv.append(i) if len(want.inv): - conn.send_message(want) + self.send_message(want) - def on_ping(self, conn, message): - if conn.ver_send > BIP0031_VERSION: - conn.send_message(msg_pong(message.nonce)) + def on_ping(self, message): + self.send_message(msg_pong(message.nonce)) - def on_verack(self, conn, message): - conn.ver_recv = conn.ver_send + def on_verack(self, message): self.verack_received = True - def on_version(self, conn, message): - if message.nVersion >= 209: - conn.send_message(msg_verack()) - conn.ver_send = min(MY_VERSION, message.nVersion) - if message.nVersion < 209: - conn.ver_recv = conn.ver_send - conn.nServices = message.nServices + def on_version(self, message): + assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_VERSION_SUPPORTED) + self.send_message(msg_verack()) + self.nServices = message.nServices # Connection helper methods - def add_connection(self, conn): - self.connection = conn - def wait_for_disconnect(self, timeout=60): - test_function = lambda: not self.connected + test_function = lambda: self.state != "connected" wait_until(test_function, timeout=timeout, lock=mininode_lock) # Message receiving helper methods @@ -1599,12 +378,6 @@ class NodeConnCB(object): # Message sending helper functions - def send_message(self, message): - if self.connection: - self.connection.send_message(message) - else: - logger.error("Cannot send message. No connection to node!") - def send_and_ping(self, message): self.send_message(message) self.sync_with_ping() @@ -1615,218 +388,25 @@ class NodeConnCB(object): test_function = lambda: self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter wait_until(test_function, timeout=timeout, lock=mininode_lock) self.ping_counter += 1 - return True - -# The actual NodeConn class -# This class provides an interface for a p2p connection to a specified node -class NodeConn(asyncore.dispatcher): - messagemap = { - b"version": msg_version, - b"verack": msg_verack, - b"addr": msg_addr, - b"alert": msg_alert, - b"inv": msg_inv, - b"getdata": msg_getdata, - b"getblocks": msg_getblocks, - b"tx": msg_tx, - b"block": msg_block, - b"getaddr": msg_getaddr, - b"ping": msg_ping, - b"pong": msg_pong, - b"headers": msg_headers, - b"getheaders": msg_getheaders, - b"reject": msg_reject, - b"mempool": msg_mempool, - b"feefilter": msg_feefilter, - b"sendheaders": msg_sendheaders, - b"sendcmpct": msg_sendcmpct, - b"cmpctblock": msg_cmpctblock, - b"getblocktxn": msg_getblocktxn, - b"blocktxn": msg_blocktxn - } - MAGIC_BYTES = { - "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet - "testnet3": b"\x0b\x11\x09\x07", # testnet3 - "regtest": b"\xfa\xbf\xb5\xda", # regtest - } - - def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK, send_version=True): - asyncore.dispatcher.__init__(self, map=mininode_socket_map) - self.dstaddr = dstaddr - self.dstport = dstport - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - self.sendbuf = b"" - self.recvbuf = b"" - self.ver_send = 209 - self.ver_recv = 209 - self.last_sent = 0 - self.state = "connecting" - self.network = net - self.cb = callback - self.disconnect = False - self.nServices = 0 - - if send_version: - # stuff version msg into sendbuf - vt = msg_version() - vt.nServices = services - vt.addrTo.ip = self.dstaddr - vt.addrTo.port = self.dstport - vt.addrFrom.ip = "0.0.0.0" - vt.addrFrom.port = 0 - self.send_message(vt, True) - logger.info('Connecting to Bitcoin Node: %s:%d' % (self.dstaddr, self.dstport)) - try: - self.connect((dstaddr, dstport)) - except: - self.handle_close() - self.rpc = rpc - - def handle_connect(self): - if self.state != "connected": - logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport)) - self.state = "connected" - self.cb.on_open(self) - - def handle_close(self): - logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport)) - self.state = "closed" - self.recvbuf = b"" - self.sendbuf = b"" - try: - self.close() - except: - pass - self.cb.on_close(self) - - def handle_read(self): - t = self.recv(8192) - if len(t) > 0: - self.recvbuf += t - self.got_data() - - def readable(self): - return True - - def writable(self): - with mininode_lock: - pre_connection = self.state == "connecting" - length = len(self.sendbuf) - return (length > 0 or pre_connection) - - def handle_write(self): - with mininode_lock: - # asyncore does not expose socket connection, only the first read/write - # event, thus we must check connection manually here to know when we - # actually connect - if self.state == "connecting": - self.handle_connect() - if not self.writable(): - return - - try: - sent = self.send(self.sendbuf) - except: - self.handle_close() - return - self.sendbuf = self.sendbuf[sent:] - - def got_data(self): - try: - while True: - if len(self.recvbuf) < 4: - return - if self.recvbuf[:4] != self.MAGIC_BYTES[self.network]: - raise ValueError("got garbage %s" % repr(self.recvbuf)) - if self.ver_recv < 209: - if len(self.recvbuf) < 4 + 12 + 4: - return - command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] - msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] - checksum = None - if len(self.recvbuf) < 4 + 12 + 4 + msglen: - return - msg = self.recvbuf[4+12+4:4+12+4+msglen] - self.recvbuf = self.recvbuf[4+12+4+msglen:] - else: - if len(self.recvbuf) < 4 + 12 + 4 + 4: - return - command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] - msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] - checksum = self.recvbuf[4+12+4:4+12+4+4] - if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen: - return - msg = self.recvbuf[4+12+4+4:4+12+4+4+msglen] - th = sha256(msg) - h = sha256(th) - if checksum != h[:4]: - raise ValueError("got bad checksum " + repr(self.recvbuf)) - self.recvbuf = self.recvbuf[4+12+4+4+msglen:] - if command in self.messagemap: - f = BytesIO(msg) - t = self.messagemap[command]() - t.deserialize(f) - self.got_message(t) - else: - logger.warning("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) - raise ValueError("Unknown command: '%s'" % (command)) - except Exception as e: - logger.exception('got_data:', repr(e)) - raise - - def send_message(self, message, pushbuf=False): - if self.state != "connected" and not pushbuf: - raise IOError('Not connected, no pushbuf') - self._log_message("send", message) - command = message.command - data = message.serialize() - tmsg = self.MAGIC_BYTES[self.network] - tmsg += command - tmsg += b"\x00" * (12 - len(command)) - tmsg += struct.pack("<I", len(data)) - if self.ver_send >= 209: - th = sha256(data) - h = sha256(th) - tmsg += h[:4] - tmsg += data - with mininode_lock: - if (len(self.sendbuf) == 0 and not pushbuf): - try: - sent = self.send(tmsg) - self.sendbuf = tmsg[sent:] - except BlockingIOError: - self.sendbuf = tmsg - else: - self.sendbuf += tmsg - self.last_sent = time.time() - - def got_message(self, message): - if message.command == b"version": - if message.nVersion <= BIP0031_VERSION: - self.messagemap[b'ping'] = msg_ping_prebip31 - if self.last_sent + 30 * 60 < time.time(): - self.send_message(self.messagemap[b'ping']()) - self._log_message("receive", message) - self.cb.deliver(self, message) - - def _log_message(self, direction, msg): - if direction == "send": - log_message = "Send message to " - elif direction == "receive": - log_message = "Received message from " - log_message += "%s:%d: %s" % (self.dstaddr, self.dstport, repr(msg)[:500]) - if len(log_message) > 500: - log_message += "... (msg truncated)" - logger.debug(log_message) +# Keep our own socket map for asyncore, so that we can track disconnects +# ourselves (to workaround an issue with closing an asyncore socket when +# using select) +mininode_socket_map = dict() - def disconnect_node(self): - self.disconnect = True +# One lock for synchronizing all data access between the networking thread (see +# NetworkThread below) and the thread running the test logic. For simplicity, +# P2PConnection acquires this lock whenever delivering a message to a P2PInterface, +# and whenever adding anything to the send buffer (in send_message()). This +# lock should be acquired in the thread running the test logic to synchronize +# access to any data shared with the P2PInterface or P2PConnection. +mininode_lock = threading.RLock() +class NetworkThread(threading.Thread): + def __init__(self): + super().__init__(name="NetworkThread") -class NetworkThread(Thread): def run(self): while mininode_socket_map: # We check for whether to disconnect outside of the asyncore @@ -1836,16 +416,27 @@ class NetworkThread(Thread): for fd, obj in mininode_socket_map.items(): if obj.disconnect: disconnected.append(obj) - [ obj.handle_close() for obj in disconnected ] + [obj.handle_close() for obj in disconnected] asyncore.loop(0.1, use_poll=True, map=mininode_socket_map, count=1) logger.debug("Network thread closing") +def network_thread_start(): + """Start the network thread.""" + # Only one network thread may run at a time + assert not network_thread_running() + + NetworkThread().start() + +def network_thread_running(): + """Return whether the network thread is running.""" + return any([thread.name == "NetworkThread" for thread in threading.enumerate()]) -# An exception we can raise if we detect a potential disconnect -# (p2p or rpc) before the test is complete -class EarlyDisconnectError(Exception): - def __init__(self, value): - self.value = value +def network_thread_join(timeout=10): + """Wait timeout seconds for the network thread to terminate. - def __str__(self): - return repr(self.value) + Throw if the network thread doesn't terminate in timeout seconds.""" + network_threads = [thread for thread in threading.enumerate() if thread.name == "NetworkThread"] + assert len(network_threads) <= 1 + for thread in network_threads: + thread.join(timeout) + assert not thread.is_alive() diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index e5d415788f..96fe283347 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Linux network utilities. diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 8f5339a02a..dae8a4e569 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Functionality to build scripts, as well as SignatureHash(). @@ -370,7 +370,7 @@ class CScriptTruncatedPushDataError(CScriptInvalidError): super(CScriptTruncatedPushDataError, self).__init__(msg) # This is used, eg, for blockchain heights in coinbase scripts (bip34) -class CScriptNum(object): +class CScriptNum(): def __init__(self, d=0): self.value = d @@ -641,7 +641,7 @@ def SignatureHash(script, txTo, inIdx, hashtype): txtmp.vin = [] txtmp.vin.append(tmp) - s = txtmp.serialize() + s = txtmp.serialize_without_witness() s += struct.pack(b"<I", hashtype) hash = hash256(s) diff --git a/test/functional/test_framework/siphash.py b/test/functional/test_framework/siphash.py index f68ecad36b..6ffc982cea 100644 --- a/test/functional/test_framework/siphash.py +++ b/test/functional/test_framework/siphash.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Specialized SipHash-2-4 implementations. diff --git a/test/functional/test_framework/socks5.py b/test/functional/test_framework/socks5.py index 0070844168..4721809a3b 100644 --- a/test/functional/test_framework/socks5.py +++ b/test/functional/test_framework/socks5.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Dummy Socks5 server for testing.""" @@ -31,7 +31,7 @@ def recvall(s, n): return rv ### Implementation classes -class Socks5Configuration(object): +class Socks5Configuration(): """Proxy configuration.""" def __init__(self): self.addr = None # Bind address (must be set) @@ -39,7 +39,7 @@ class Socks5Configuration(object): self.unauth = False # Support unauthenticated self.auth = False # Support authentication -class Socks5Command(object): +class Socks5Command(): """Information about an incoming socks5 command.""" def __init__(self, cmd, atyp, addr, port, username, password): self.cmd = cmd # Command (one of Command.*) @@ -51,7 +51,7 @@ class Socks5Command(object): def __repr__(self): return 'Socks5Command(%s,%s,%s,%s,%s,%s)' % (self.cmd, self.atyp, self.addr, self.port, self.username, self.password) -class Socks5Connection(object): +class Socks5Connection(): def __init__(self, serv, conn, peer): self.serv = serv self.conn = conn @@ -122,7 +122,7 @@ class Socks5Connection(object): finally: self.conn.close() -class Socks5Server(object): +class Socks5Server(): def __init__(self, conf): self.conf = conf self.s = socket.socket(conf.af) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 381513ab9e..5d4f8e6720 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,10 +1,9 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" -from collections import deque from enum import Enum import logging import optparse @@ -14,7 +13,6 @@ import shutil import sys import tempfile import time -import traceback from .authproxy import JSONRPCException from . import coverage @@ -43,7 +41,7 @@ TEST_EXIT_PASSED = 0 TEST_EXIT_FAILED = 1 TEST_EXIT_SKIPPED = 77 -class BitcoinTestFramework(object): +class BitcoinTestFramework(): """Base class for a bitcoin test script. Individual bitcoin test scripts should subclass this class and override the set_test_params() and run_test() methods. @@ -64,6 +62,7 @@ class BitcoinTestFramework(object): self.setup_clean_chain = False self.nodes = [] self.mocktime = 0 + self.supports_cli = False self.set_test_params() assert hasattr(self, "num_nodes"), "Test must set self.num_nodes in set_test_params()" @@ -93,6 +92,8 @@ class BitcoinTestFramework(object): help="Location of the test framework config file") parser.add_option("--pdbonfailure", dest="pdbonfailure", default=False, action="store_true", help="Attach a python debugger if test fails") + parser.add_option("--usecli", dest="usecli", default=False, action="store_true", + help="use bitcoin-cli instead of RPC for all commands") self.add_options(parser) (self.options, self.args) = parser.parse_args() @@ -102,8 +103,11 @@ class BitcoinTestFramework(object): check_json_precision() + self.options.cachedir = os.path.abspath(self.options.cachedir) + # Set up temp directory and start logging if self.options.tmpdir: + self.options.tmpdir = os.path.abspath(self.options.tmpdir) os.makedirs(self.options.tmpdir, exist_ok=False) else: self.options.tmpdir = tempfile.mkdtemp(prefix="test") @@ -112,6 +116,8 @@ class BitcoinTestFramework(object): success = TestStatus.FAILED try: + if self.options.usecli and not self.supports_cli: + raise SkipTest("--usecli specified but test does not support using CLI") self.setup_chain() self.setup_network() self.run_test() @@ -146,32 +152,19 @@ class BitcoinTestFramework(object): shutil.rmtree(self.options.tmpdir) else: self.log.warning("Not cleaning up dir %s" % self.options.tmpdir) - if os.getenv("PYTHON_DEBUG", ""): - # Dump the end of the debug logs, to aid in debugging rare - # travis failures. - import glob - filenames = [self.options.tmpdir + "/test_framework.log"] - filenames += glob.glob(self.options.tmpdir + "/node*/regtest/debug.log") - MAX_LINES_TO_PRINT = 1000 - for fn in filenames: - try: - with open(fn, 'r') as f: - print("From", fn, ":") - print("".join(deque(f, MAX_LINES_TO_PRINT))) - except OSError: - print("Opening file %s failed." % fn) - traceback.print_exc() if success == TestStatus.PASSED: self.log.info("Tests successful") - sys.exit(TEST_EXIT_PASSED) + exit_code = TEST_EXIT_PASSED elif success == TestStatus.SKIPPED: self.log.info("Test skipped") - sys.exit(TEST_EXIT_SKIPPED) + exit_code = TEST_EXIT_SKIPPED else: self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) - logging.shutdown() - sys.exit(TEST_EXIT_FAILED) + self.log.error("Hint: Call {} '{}' to consolidate all logs".format(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + "/../combine_logs.py"), self.options.tmpdir)) + exit_code = TEST_EXIT_FAILED + logging.shutdown() + sys.exit(exit_code) # Methods to override in subclass test scripts. def set_test_params(self): @@ -225,7 +218,7 @@ class BitcoinTestFramework(object): assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) for i in range(num_nodes): - self.nodes.append(TestNode(i, self.options.tmpdir, extra_args[i], rpchost, timewait=timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir)) + self.nodes.append(TestNode(i, self.options.tmpdir, extra_args[i], rpchost, timewait=timewait, binary=binary[i], stderr=None, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, use_cli=self.options.usecli)) def start_node(self, i, extra_args=None, stderr=None): """Start a bitcoind""" @@ -429,7 +422,7 @@ class BitcoinTestFramework(object): self.disable_mocktime() for i in range(MAX_NODES): os.remove(log_filename(self.options.cachedir, i, "debug.log")) - os.remove(log_filename(self.options.cachedir, i, "db.log")) + os.remove(log_filename(self.options.cachedir, i, "wallets/db.log")) os.remove(log_filename(self.options.cachedir, i, "peers.dat")) os.remove(log_filename(self.options.cachedir, i, "fee_estimates.dat")) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 12dab57a02..589a8f3969 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -10,16 +10,21 @@ import http.client import json import logging import os +import re import subprocess import time +from .authproxy import JSONRPCException from .util import ( assert_equal, get_rpc_proxy, rpc_url, wait_until, + p2p_port, ) -from .authproxy import JSONRPCException + +# For Python 3.4 compatibility +JSONDecodeError = getattr(json, "JSONDecodeError", ValueError) BITCOIND_PROC_WAIT_TIMEOUT = 60 @@ -31,11 +36,13 @@ class TestNode(): - state about the node (whether it's running, etc) - a Python subprocess.Popen object representing the running process - an RPC connection to the node + - one or more P2P connections to the node - To make things easier for the test writer, a bit of magic is happening under the covers. - Any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mocktime, coverage_dir): + To make things easier for the test writer, any unrecognised messages will + be dispatched to the RPC connection.""" + + def __init__(self, i, dirname, extra_args, rpchost, timewait, binary, stderr, mocktime, coverage_dir, use_cli=False): self.index = i self.datadir = os.path.join(dirname, "node" + str(i)) self.rpchost = rpchost @@ -55,6 +62,7 @@ class TestNode(): self.args = [self.binary, "-datadir=" + self.datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(mocktime), "-uacomment=testnode%d" % i] self.cli = TestNodeCLI(os.getenv("BITCOINCLI", "bitcoin-cli"), self.datadir) + self.use_cli = use_cli self.running = False self.process = None @@ -63,10 +71,15 @@ class TestNode(): self.url = None self.log = logging.getLogger('TestFramework.node%d' % i) - def __getattr__(self, *args, **kwargs): - """Dispatches any unrecognised messages to the RPC connection.""" - assert self.rpc_connected and self.rpc is not None, "Error: no RPC connection" - return self.rpc.__getattr__(*args, **kwargs) + self.p2ps = [] + + def __getattr__(self, name): + """Dispatches any unrecognised messages to the RPC connection or a CLI instance.""" + if self.use_cli: + return getattr(self.cli, name) + else: + assert self.rpc_connected and self.rpc is not None, "Error: no RPC connection" + return getattr(self.rpc, name) def start(self, extra_args=None, stderr=None): """Start the node.""" @@ -105,10 +118,13 @@ class TestNode(): raise AssertionError("Unable to connect to bitcoind") def get_wallet_rpc(self, wallet_name): - assert self.rpc_connected - assert self.rpc - wallet_path = "wallet/%s" % wallet_name - return self.rpc / wallet_path + if self.use_cli: + return self.cli("-rpcwallet={}".format(wallet_name)) + else: + assert self.rpc_connected + assert self.rpc + wallet_path = "wallet/%s" % wallet_name + return self.rpc / wallet_path def stop_node(self): """Stop the node.""" @@ -119,6 +135,7 @@ class TestNode(): self.stop() except http.client.CannotSendRequest: self.log.exception("Unable to stop node.") + del self.p2ps[:] def is_node_stopped(self): """Checks whether the node has stopped. @@ -151,6 +168,47 @@ class TestNode(): self.encryptwallet(passphrase) self.wait_until_stopped() + def add_p2p_connection(self, p2p_conn, *args, **kwargs): + """Add a p2p connection to the node. + + This method adds the p2p connection to the self.p2ps list and also + returns the connection to the caller.""" + if 'dstport' not in kwargs: + kwargs['dstport'] = p2p_port(self.index) + if 'dstaddr' not in kwargs: + kwargs['dstaddr'] = '127.0.0.1' + + p2p_conn.peer_connect(*args, **kwargs) + self.p2ps.append(p2p_conn) + + return p2p_conn + + @property + def p2p(self): + """Return the first p2p connection + + Convenience property - most tests only use a single p2p connection to each + node, so this saves having to write node.p2ps[0] many times.""" + assert self.p2ps, "No p2p connection" + return self.p2ps[0] + + def disconnect_p2ps(self): + """Close all p2p connections to the node.""" + for p in self.p2ps: + p.peer_disconnect() + del self.p2ps[:] + +class TestNodeCLIAttr: + def __init__(self, cli, command): + self.cli = cli + self.command = command + + def __call__(self, *args, **kwargs): + return self.cli.send_cli(self.command, *args, **kwargs) + + def get_request(self, *args, **kwargs): + return lambda: self(*args, **kwargs) + class TestNodeCLI(): """Interface to bitcoin-cli for an individual node""" @@ -159,17 +217,26 @@ class TestNodeCLI(): self.binary = binary self.datadir = datadir self.input = None + self.log = logging.getLogger('TestFramework.bitcoincli') def __call__(self, *args, input=None): # TestNodeCLI is callable with bitcoin-cli command-line args - self.args = [str(arg) for arg in args] - self.input = input - return self + cli = TestNodeCLI(self.binary, self.datadir) + cli.args = [str(arg) for arg in args] + cli.input = input + return cli def __getattr__(self, command): - def dispatcher(*args, **kwargs): - return self.send_cli(command, *args, **kwargs) - return dispatcher + return TestNodeCLIAttr(self, command) + + def batch(self, requests): + results = [] + for request in requests: + try: + results.append(dict(result=request())) + except JSONRPCException as e: + results.append(dict(error=e)) + return results def send_cli(self, command, *args, **kwargs): """Run bitcoin-cli command. Deserializes returned string as python object.""" @@ -181,10 +248,18 @@ class TestNodeCLI(): if named_args: p_args += ["-named"] p_args += [command] + pos_args + named_args + self.log.debug("Running bitcoin-cli command: %s" % command) process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) cli_stdout, cli_stderr = process.communicate(input=self.input) returncode = process.poll() if returncode: + match = re.match(r'error code: ([-0-9]+)\nerror message:\n(.*)', cli_stderr) + if match: + code, message = match.groups() + raise JSONRPCException(dict(code=int(code), message=message)) # Ignore cli_stdout, raise with cli_stderr raise subprocess.CalledProcessError(returncode, self.binary, output=cli_stderr) - return json.loads(cli_stdout, parse_float=decimal.Decimal) + try: + return json.loads(cli_stdout, parse_float=decimal.Decimal) + except JSONDecodeError: + return cli_stdout.rstrip("\n") diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 102c903018..664dc6d0af 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helpful routines for regression testing.""" diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index e85d672096..72ad300e7e 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run regression test suite. @@ -15,6 +15,7 @@ For a description of arguments recognized by test scripts, see """ import argparse +from collections import deque import configparser import datetime import os @@ -77,6 +78,7 @@ BASE_SCRIPTS= [ 'abandonconflict.py', 'bip68-112-113-p2p.py', 'rawtransactions.py', + 'address_types.py', 'reindex.py', # vv Tests less than 30s vv 'keypool-topup.py', @@ -85,12 +87,14 @@ BASE_SCRIPTS= [ 'mempool_resurrect_test.py', 'txn_doublespend.py --mineblock', 'txn_clone.py', + 'txn_clone.py --segwit', 'getchaintips.py', 'rest.py', 'mempool_spendcoinbase.py', 'mempool_reorg.py', 'mempool_persist.py', 'multiwallet.py', + 'multiwallet.py --usecli', 'httpbasics.py', 'multi_rpc.py', 'proxy_test.py', @@ -123,6 +127,14 @@ BASE_SCRIPTS= [ 'uptime.py', 'resendwallettransactions.py', 'minchainwork.py', + 'p2p-fingerprint.py', + 'uacomment.py', + 'p2p-acceptblock.py', + 'feature_logging.py', + 'node_network_limited.py', + 'conf_args.py', + # Don't append tests at the end to avoid merge conflicts + # Put them in a random line within the section that fits their approximate run-time ] EXTENDED_SCRIPTS = [ @@ -148,9 +160,8 @@ EXTENDED_SCRIPTS = [ 'example_test.py', 'txn_doublespend.py', 'txn_clone.py --mineblock', - 'forknotify.py', + 'notifications.py', 'invalidateblock.py', - 'p2p-acceptblock.py', 'replace-by-fee.py', ] @@ -172,6 +183,7 @@ def main(): epilog=''' Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('--combinedlogslen', '-c', type=int, default=0, help='print a combined log (of length n lines) from all test nodes and test framework to the console on failure.') parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') @@ -265,9 +277,9 @@ def main(): if not args.keepcache: shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) - run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args) + run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, args.combinedlogslen) -def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]): +def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[], combined_logs_len=0): # Warn if bitcoind is already running (unix only) try: if subprocess.check_output(["pidof", "bitcoind"]) is not None: @@ -299,7 +311,11 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove if len(test_list) > 1 and jobs > 1: # Populate cache - subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) + try: + subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) + except Exception as e: + print(e.output) + raise e #Run Tests job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) @@ -309,7 +325,7 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove max_len_name = len(max(test_list, key=len)) for _ in range(len(test_list)): - test_result, stdout, stderr = job_queue.get_next() + test_result, testdir, stdout, stderr = job_queue.get_next() test_results.append(test_result) if test_result.status == "Passed": @@ -320,6 +336,14 @@ def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_cove print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time)) print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n') print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n') + if combined_logs_len and os.path.isdir(testdir): + # Print the final `combinedlogslen` lines of the combined logs + print('{}Combine the logs and print the last {} lines ...{}'.format(BOLD[1], combined_logs_len, BOLD[0])) + print('\n============') + print('{}Combined log for {}:{}'.format(BOLD[1], testdir, BOLD[0])) + print('============\n') + combined_logs, _ = subprocess.Popen([os.path.join(tests_dir, 'combine_logs.py'), '-c', testdir], universal_newlines=True, stdout=subprocess.PIPE).communicate() + print("\n".join(deque(combined_logs.splitlines(), combined_logs_len))) print_results(test_results, max_len_name, (int(time.time() - time0))) @@ -384,13 +408,15 @@ class TestHandler: log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) test_argv = t.split() - tmpdir = ["--tmpdir=%s/%s_%s" % (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)] + testdir = "{}/{}_{}".format(self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed) + tmpdir_arg = ["--tmpdir={}".format(testdir)] self.jobs.append((t, time.time(), - subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir, + subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir_arg, universal_newlines=True, stdout=log_stdout, stderr=log_stderr), + testdir, log_stdout, log_stderr)) if not self.jobs: @@ -399,7 +425,7 @@ class TestHandler: # Return first proc that finishes time.sleep(.5) for j in self.jobs: - (name, time0, proc, log_out, log_err) = j + (name, time0, proc, testdir, log_out, log_err) = j if os.getenv('TRAVIS') == 'true' and int(time.time() - time0) > 20 * 60: # In travis, timeout individual tests after 20 minutes (to stop tests hanging and not # providing useful output. @@ -417,7 +443,7 @@ class TestHandler: self.num_running -= 1 self.jobs.remove(j) - return TestResult(name, status, int(time.time() - time0)), stdout, stderr + return TestResult(name, status, int(time.time() - time0)), testdir, stdout, stderr print('.', end='', flush=True) class TestResult(): @@ -481,7 +507,7 @@ def check_script_list(src_dir): # On travis this warning is an error to prevent merging incomplete commits into master sys.exit(1) -class RPCCoverage(object): +class RPCCoverage(): """ Coverage reporting utilities for test_runner. diff --git a/test/functional/txn_clone.py b/test/functional/txn_clone.py index 740bb2d4c5..ce26d6e0ee 100755 --- a/test/functional/txn_clone.py +++ b/test/functional/txn_clone.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there are cloned transactions with malleated scriptsigs.""" @@ -14,6 +14,8 @@ class TxnMallTest(BitcoinTestFramework): def add_options(self, parser): parser.add_option("--mineblock", dest="mine_block", default=False, action="store_true", help="Test double-spend of 1-confirmed transaction") + parser.add_option("--segwit", dest="segwit", default=False, action="store_true", + help="Test behaviour with SegWit txn (which should fail") def setup_network(self): # Start with split network: @@ -22,6 +24,11 @@ class TxnMallTest(BitcoinTestFramework): disconnect_nodes(self.nodes[2], 1) def run_test(self): + if self.options.segwit: + output_type="p2sh-segwit" + else: + output_type="legacy" + # All nodes should start with 1,250 BTC: starting_balance = 1250 for i in range(4): @@ -31,11 +38,11 @@ class TxnMallTest(BitcoinTestFramework): # Assign coins to foo and bar accounts: self.nodes[0].settxfee(.001) - node0_address_foo = self.nodes[0].getnewaddress("foo") + node0_address_foo = self.nodes[0].getnewaddress("foo", output_type) fund_foo_txid = self.nodes[0].sendfrom("", node0_address_foo, 1219) fund_foo_tx = self.nodes[0].gettransaction(fund_foo_txid) - node0_address_bar = self.nodes[0].getnewaddress("bar") + node0_address_bar = self.nodes[0].getnewaddress("bar", output_type) fund_bar_txid = self.nodes[0].sendfrom("", node0_address_bar, 29) fund_bar_tx = self.nodes[0].gettransaction(fund_bar_txid) @@ -106,6 +113,10 @@ class TxnMallTest(BitcoinTestFramework): # Send clone and its parent to miner self.nodes[2].sendrawtransaction(fund_foo_tx["hex"]) txid1_clone = self.nodes[2].sendrawtransaction(tx1_clone["hex"]) + if self.options.segwit: + assert_equal(txid1, txid1_clone) + return + # ... mine a block... self.nodes[2].generate(1) diff --git a/test/functional/txn_doublespend.py b/test/functional/txn_doublespend.py index 69629ef951..01129f3817 100755 --- a/test/functional/txn_doublespend.py +++ b/test/functional/txn_doublespend.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there is a double-spend conflict.""" diff --git a/test/functional/uacomment.py b/test/functional/uacomment.py new file mode 100755 index 0000000000..0b2c64ab69 --- /dev/null +++ b/test/functional/uacomment.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the -uacomment option.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +class UacommentTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + self.log.info("test multiple -uacomment") + test_uacomment = self.nodes[0].getnetworkinfo()["subversion"][-12:-1] + assert_equal(test_uacomment, "(testnode0)") + + self.restart_node(0, ["-uacomment=foo"]) + foo_uacomment = self.nodes[0].getnetworkinfo()["subversion"][-17:-1] + assert_equal(foo_uacomment, "(testnode0; foo)") + + self.log.info("test -uacomment max length") + self.stop_node(0) + expected = "Total length of network version string (286) exceeds maximum length (256). Reduce the number or size of uacomments." + self.assert_start_raises_init_error(0, ["-uacomment=" + 'a' * 256], expected) + + self.log.info("test -uacomment unsafe characters") + for unsafe_char in ['/', ':', '(', ')']: + expected = "User Agent comment (" + unsafe_char + ") contains unsafe characters" + self.assert_start_raises_init_error(0, ["-uacomment=" + unsafe_char], expected) + +if __name__ == '__main__': + UacommentTest().main() diff --git a/test/functional/wallet-accounts.py b/test/functional/wallet-accounts.py index 40726d2a76..287436877f 100755 --- a/test/functional/wallet-accounts.py +++ b/test/functional/wallet-accounts.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test account RPCs. @@ -72,62 +72,135 @@ class WalletAccountsTest(BitcoinTestFramework): # otherwise we're off by exactly the fee amount as that's mined # and matures in the next 100 blocks node.sendfrom("", common_address, fee) - accounts = ["a", "b", "c", "d", "e"] amount_to_send = 1.0 - account_addresses = dict() + + # Create accounts and make sure subsequent account API calls + # recognize the account/address associations. + accounts = [Account(name) for name in ("a", "b", "c", "d", "e")] for account in accounts: - address = node.getaccountaddress(account) - account_addresses[account] = address - - node.getnewaddress(account) - assert_equal(node.getaccount(address), account) - assert(address in node.getaddressesbyaccount(account)) - - node.sendfrom("", address, amount_to_send) - + account.add_receive_address(node.getaccountaddress(account.name)) + account.verify(node) + + # Send a transaction to each account, and make sure this forces + # getaccountaddress to generate a new receiving address. + for account in accounts: + node.sendtoaddress(account.receive_address, amount_to_send) + account.add_receive_address(node.getaccountaddress(account.name)) + account.verify(node) + + # Check the amounts received. node.generate(1) + for account in accounts: + assert_equal( + node.getreceivedbyaddress(account.addresses[0]), amount_to_send) + assert_equal(node.getreceivedbyaccount(account.name), amount_to_send) - for i in range(len(accounts)): - from_account = accounts[i] + # Check that sendfrom account reduces listaccounts balances. + for i, account in enumerate(accounts): to_account = accounts[(i+1) % len(accounts)] - to_address = account_addresses[to_account] - node.sendfrom(from_account, to_address, amount_to_send) - + node.sendfrom(account.name, to_account.receive_address, amount_to_send) node.generate(1) - for account in accounts: - address = node.getaccountaddress(account) - assert(address != account_addresses[account]) - assert_equal(node.getreceivedbyaccount(account), 2) - node.move(account, "", node.getbalance(account)) - + account.add_receive_address(node.getaccountaddress(account.name)) + account.verify(node) + assert_equal(node.getreceivedbyaccount(account.name), 2) + node.move(account.name, "", node.getbalance(account.name)) + account.verify(node) node.generate(101) - expected_account_balances = {"": 5200} for account in accounts: - expected_account_balances[account] = 0 - + expected_account_balances[account.name] = 0 assert_equal(node.listaccounts(), expected_account_balances) - assert_equal(node.getbalance(""), 5200) + # Check that setaccount can assign an account to a new unused address. for account in accounts: address = node.getaccountaddress("") - node.setaccount(address, account) - assert(address in node.getaddressesbyaccount(account)) + node.setaccount(address, account.name) + account.add_address(address) + account.verify(node) assert(address not in node.getaddressesbyaccount("")) + # Check that addmultisigaddress can assign accounts. for account in accounts: addresses = [] for x in range(10): addresses.append(node.getnewaddress()) - multisig_address = node.addmultisigaddress(5, addresses, account) + multisig_address = node.addmultisigaddress(5, addresses, account.name) + account.add_address(multisig_address) + account.verify(node) node.sendfrom("", multisig_address, 50) - node.generate(101) - for account in accounts: - assert_equal(node.getbalance(account), 50) + assert_equal(node.getbalance(account.name), 50) + + # Check that setaccount can change the account of an address from a + # different account. + change_account(node, accounts[0].addresses[0], accounts[0], accounts[1]) + + # Check that setaccount can change the account of an address which + # is the receiving address of a different account. + change_account(node, accounts[0].receive_address, accounts[0], accounts[1]) + + # Check that setaccount can set the account of an address already + # in the account. This is a no-op. + change_account(node, accounts[2].addresses[0], accounts[2], accounts[2]) + + # Check that setaccount can set the account of an address which is + # already the receiving address of the account. It would probably make + # sense for this to be a no-op, but right now it resets the receiving + # address, causing getaccountaddress to return a brand new address. + change_account(node, accounts[2].receive_address, accounts[2], accounts[2]) + +class Account: + def __init__(self, name): + # Account name + self.name = name + # Current receiving address associated with this account. + self.receive_address = None + # List of all addresses assigned with this account + self.addresses = [] + + def add_address(self, address): + assert_equal(address not in self.addresses, True) + self.addresses.append(address) + + def add_receive_address(self, address): + self.add_address(address) + self.receive_address = address + + def verify(self, node): + if self.receive_address is not None: + assert self.receive_address in self.addresses + assert_equal(node.getaccountaddress(self.name), self.receive_address) + + for address in self.addresses: + assert_equal(node.getaccount(address), self.name) + + assert_equal( + set(node.getaddressesbyaccount(self.name)), set(self.addresses)) + + +def change_account(node, address, old_account, new_account): + assert_equal(address in old_account.addresses, True) + node.setaccount(address, new_account.name) + + old_account.addresses.remove(address) + new_account.add_address(address) + + # Calling setaccount on an address which was previously the receiving + # address of a different account should reset the receiving address of + # the old account, causing getaccountaddress to return a brand new + # address. + if address == old_account.receive_address: + new_address = node.getaccountaddress(old_account.name) + assert_equal(new_address not in old_account.addresses, True) + assert_equal(new_address not in new_account.addresses, True) + old_account.add_receive_address(new_address) + + old_account.verify(node) + new_account.verify(node) + if __name__ == '__main__': WalletAccountsTest().main() diff --git a/test/functional/wallet-dump.py b/test/functional/wallet-dump.py index 47de8777a6..8b81c81ba8 100755 --- a/test/functional/wallet-dump.py +++ b/test/functional/wallet-dump.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" @@ -10,13 +10,14 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import (assert_equal, assert_raises_rpc_error) -def read_dump(file_name, addrs, hd_master_addr_old): +def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): """ Read the given dump, count the addrs that match, count change and reserve. Also check that the old hd_master is inactive """ with open(file_name, encoding='utf8') as inputfile: found_addr = 0 + found_script_addr = 0 found_addr_chg = 0 found_addr_rsv = 0 hd_master_addr_ret = None @@ -38,6 +39,9 @@ def read_dump(file_name, addrs, hd_master_addr_old): # ensure we have generated a new hd master key assert(hd_master_addr_old != addr) hd_master_addr_ret = addr + elif keytype == "script=1": + # scripts don't have keypaths + keypath = None else: keypath = addr_keypath.rstrip().split("hdkeypath=")[1] @@ -52,13 +56,20 @@ def read_dump(file_name, addrs, hd_master_addr_old): elif keytype == "reserve=1": found_addr_rsv += 1 break - return found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret + + # count scripts + for script_addr in script_addrs: + if script_addr == addr.rstrip() and keytype == "script=1": + found_script_addr += 1 + break + + return found_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret class WalletDumpTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.extra_args = [["-keypool=90"]] + self.extra_args = [["-keypool=90", "-addresstype=legacy"]] def setup_network(self, split=False): # Use 1 minute timeout because the initial getnewaddress RPC can take @@ -81,13 +92,19 @@ class WalletDumpTest(BitcoinTestFramework): # Should be a no-op: self.nodes[0].keypoolrefill() + # Test scripts dump by adding a P2SH witness and a 1-of-1 multisig address + witness_addr = self.nodes[0].addwitnessaddress(addrs[0]["address"], True) + multisig_addr = self.nodes[0].addmultisigaddress(1, [addrs[1]["address"]]) + script_addrs = [witness_addr, multisig_addr] + # dump unencrypted wallet result = self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.unencrypted.dump") assert_equal(result['filename'], os.path.abspath(tmpdir + "/node0/wallet.unencrypted.dump")) - found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ - read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, None) + found_addr, found_script_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ + read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, script_addrs, None) assert_equal(found_addr, test_addr_count) # all keys must be in the dump + assert_equal(found_script_addr, 2) # all scripts must be in the dump assert_equal(found_addr_chg, 50) # 50 blocks where mined assert_equal(found_addr_rsv, 90*2) # 90 keys plus 100% internal keys @@ -99,14 +116,29 @@ class WalletDumpTest(BitcoinTestFramework): self.nodes[0].keypoolrefill() self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.encrypted.dump") - found_addr, found_addr_chg, found_addr_rsv, _ = \ - read_dump(tmpdir + "/node0/wallet.encrypted.dump", addrs, hd_master_addr_unenc) + found_addr, found_script_addr, found_addr_chg, found_addr_rsv, _ = \ + read_dump(tmpdir + "/node0/wallet.encrypted.dump", addrs, script_addrs, hd_master_addr_unenc) assert_equal(found_addr, test_addr_count) + assert_equal(found_script_addr, 2) assert_equal(found_addr_chg, 90*2 + 50) # old reserve keys are marked as change now assert_equal(found_addr_rsv, 90*2) # Overwriting should fail assert_raises_rpc_error(-8, "already exists", self.nodes[0].dumpwallet, tmpdir + "/node0/wallet.unencrypted.dump") + # Restart node with new wallet, and test importwallet + self.stop_node(0) + self.start_node(0, ['-wallet=w2']) + + # Make sure the address is not IsMine before import + result = self.nodes[0].validateaddress(multisig_addr) + assert(result['ismine'] == False) + + self.nodes[0].importwallet(os.path.abspath(tmpdir + "/node0/wallet.unencrypted.dump")) + + # Now check IsMine is true + result = self.nodes[0].validateaddress(multisig_addr) + assert(result['ismine'] == True) + if __name__ == '__main__': WalletDumpTest().main () diff --git a/test/functional/wallet-encryption.py b/test/functional/wallet-encryption.py index db62e1e30f..452e8ec291 100755 --- a/test/functional/wallet-encryption.py +++ b/test/functional/wallet-encryption.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Wallet encryption""" diff --git a/test/functional/wallet-hd.py b/test/functional/wallet-hd.py index 5ef3bf5bff..9f0e9acb47 100755 --- a/test/functional/wallet-hd.py +++ b/test/functional/wallet-hd.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Hierarchical Deterministic wallet function.""" @@ -10,6 +10,7 @@ from test_framework.util import ( connect_nodes_bi, ) import shutil +import os class WalletHDTest(BitcoinTestFramework): def set_test_params(self): @@ -70,9 +71,9 @@ class WalletHDTest(BitcoinTestFramework): self.stop_node(1) # we need to delete the complete regtest directory # otherwise node1 would auto-recover all funds in flag the keypool keys as used - shutil.rmtree(tmpdir + "/node1/regtest/blocks") - shutil.rmtree(tmpdir + "/node1/regtest/chainstate") - shutil.copyfile(tmpdir + "/hd.bak", tmpdir + "/node1/regtest/wallet.dat") + shutil.rmtree(os.path.join(tmpdir, "node1/regtest/blocks")) + shutil.rmtree(os.path.join(tmpdir, "node1/regtest/chainstate")) + shutil.copyfile(os.path.join(tmpdir, "hd.bak"), os.path.join(tmpdir, "node1/regtest/wallets/wallet.dat")) self.start_node(1) # Assert that derivation is deterministic @@ -91,6 +92,22 @@ class WalletHDTest(BitcoinTestFramework): self.start_node(1, extra_args=self.extra_args[1] + ['-rescan']) assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) + # Try a RPC based rescan + self.stop_node(1) + shutil.rmtree(os.path.join(tmpdir, "node1/regtest/blocks")) + shutil.rmtree(os.path.join(tmpdir, "node1/regtest/chainstate")) + shutil.copyfile(os.path.join(tmpdir, "hd.bak"), os.path.join(tmpdir, "node1/regtest/wallet.dat")) + self.start_node(1, extra_args=self.extra_args[1]) + connect_nodes_bi(self.nodes, 0, 1) + self.sync_all() + out = self.nodes[1].rescanblockchain(0, 1) + assert_equal(out['start_height'], 0) + assert_equal(out['stop_height'], 1) + out = self.nodes[1].rescanblockchain() + assert_equal(out['start_height'], 0) + assert_equal(out['stop_height'], self.nodes[1].getblockcount()) + assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) + # send a tx and make sure its using the internal chain for the changeoutput txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) outs = self.nodes[1].decoderawtransaction(self.nodes[1].gettransaction(txid)['hex'])['vout'] diff --git a/test/functional/wallet.py b/test/functional/wallet.py index 9d8ae50354..a90dbc8adf 100755 --- a/test/functional/wallet.py +++ b/test/functional/wallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet.""" @@ -27,6 +27,9 @@ class WalletTest(BitcoinTestFramework): assert_fee_amount(fee, tx_size, fee_per_byte * 1000) return curr_balance + def get_vsize(self, txn): + return self.nodes[0].decoderawtransaction(txn)['vsize'] + def run_test(self): # Check that there's no UTXO on none of the nodes assert_equal(len(self.nodes[0].listunspent()), 0) @@ -100,11 +103,19 @@ class WalletTest(BitcoinTestFramework): # Exercise locking of unspent outputs unspent_0 = self.nodes[2].listunspent()[0] unspent_0 = {"txid": unspent_0["txid"], "vout": unspent_0["vout"]} + assert_raises_rpc_error(-8, "Invalid parameter, expected locked output", self.nodes[2].lockunspent, True, [unspent_0]) self.nodes[2].lockunspent(False, [unspent_0]) + assert_raises_rpc_error(-8, "Invalid parameter, output already locked", self.nodes[2].lockunspent, False, [unspent_0]) assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) assert_equal([unspent_0], self.nodes[2].listlockunspent()) self.nodes[2].lockunspent(True, [unspent_0]) assert_equal(len(self.nodes[2].listlockunspent()), 0) + assert_raises_rpc_error(-8, "Invalid parameter, unknown transaction", + self.nodes[2].lockunspent, False, + [{"txid": "0000000000000000000000000000000000", "vout": 0}]) + assert_raises_rpc_error(-8, "Invalid parameter, vout index out of bounds", + self.nodes[2].lockunspent, False, + [{"txid": unspent_0["txid"], "vout": 999}]) # Have node1 generate 100 blocks (so node0 can recover the fee) self.nodes[1].generate(100) @@ -143,6 +154,10 @@ class WalletTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), 94) assert_equal(self.nodes[2].getbalance("from1"), 94-21) + # Verify that a spent output cannot be locked anymore + spent_0 = {"txid": node0utxos[0]["txid"], "vout": node0utxos[0]["vout"]} + assert_raises_rpc_error(-8, "Invalid parameter, expected unspent output", self.nodes[0].lockunspent, False, [spent_0]) + # Send 10 BTC normal address = self.nodes[0].getnewaddress("test") fee_per_byte = Decimal('0.001') / 1000 @@ -150,7 +165,7 @@ class WalletTest(BitcoinTestFramework): txid = self.nodes[2].sendtoaddress(address, 10, "", "", False) self.nodes[2].generate(1) self.sync_all([self.nodes[0:3]]) - node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, count_bytes(self.nodes[2].getrawtransaction(txid))) + node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) assert_equal(self.nodes[0].getbalance(), Decimal('10')) # Send 10 BTC with subtract fee from amount @@ -159,14 +174,14 @@ class WalletTest(BitcoinTestFramework): self.sync_all([self.nodes[0:3]]) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) - node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, count_bytes(self.nodes[2].getrawtransaction(txid))) + node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) # Sendmany 10 BTC txid = self.nodes[2].sendmany('from1', {address: 10}, 0, "", []) self.nodes[2].generate(1) self.sync_all([self.nodes[0:3]]) node_0_bal += Decimal('10') - node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, count_bytes(self.nodes[2].getrawtransaction(txid))) + node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) assert_equal(self.nodes[0].getbalance(), node_0_bal) # Sendmany 10 BTC with subtract fee from amount @@ -175,7 +190,7 @@ class WalletTest(BitcoinTestFramework): self.sync_all([self.nodes[0:3]]) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) - node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, count_bytes(self.nodes[2].getrawtransaction(txid))) + node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].getrawtransaction(txid))) # Test ResendWalletTransactions: # Create a couple of transactions, then start up a fourth diff --git a/test/functional/walletbackup.py b/test/functional/walletbackup.py index 15ea26afa1..b4be7debb5 100755 --- a/test/functional/walletbackup.py +++ b/test/functional/walletbackup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet backup features. @@ -90,9 +90,9 @@ class WalletBackupTest(BitcoinTestFramework): self.stop_node(2) def erase_three(self): - os.remove(self.options.tmpdir + "/node0/regtest/wallet.dat") - os.remove(self.options.tmpdir + "/node1/regtest/wallet.dat") - os.remove(self.options.tmpdir + "/node2/regtest/wallet.dat") + os.remove(self.options.tmpdir + "/node0/regtest/wallets/wallet.dat") + os.remove(self.options.tmpdir + "/node1/regtest/wallets/wallet.dat") + os.remove(self.options.tmpdir + "/node2/regtest/wallets/wallet.dat") def run_test(self): self.log.info("Generating initial blockchain") @@ -154,9 +154,9 @@ class WalletBackupTest(BitcoinTestFramework): shutil.rmtree(self.options.tmpdir + "/node2/regtest/chainstate") # Restore wallets from backup - shutil.copyfile(tmpdir + "/node0/wallet.bak", tmpdir + "/node0/regtest/wallet.dat") - shutil.copyfile(tmpdir + "/node1/wallet.bak", tmpdir + "/node1/regtest/wallet.dat") - shutil.copyfile(tmpdir + "/node2/wallet.bak", tmpdir + "/node2/regtest/wallet.dat") + shutil.copyfile(tmpdir + "/node0/wallet.bak", tmpdir + "/node0/regtest/wallets/wallet.dat") + shutil.copyfile(tmpdir + "/node1/wallet.bak", tmpdir + "/node1/regtest/wallets/wallet.dat") + shutil.copyfile(tmpdir + "/node2/wallet.bak", tmpdir + "/node2/regtest/wallets/wallet.dat") self.log.info("Re-starting nodes") self.start_three() @@ -190,6 +190,16 @@ class WalletBackupTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), balance1) assert_equal(self.nodes[2].getbalance(), balance2) + # Backup to source wallet file must fail + sourcePaths = [ + tmpdir + "/node0/regtest/wallets/wallet.dat", + tmpdir + "/node0/./regtest/wallets/wallet.dat", + tmpdir + "/node0/regtest/wallets/", + tmpdir + "/node0/regtest/wallets"] + + for sourcePath in sourcePaths: + assert_raises_rpc_error(-4, "backup failed", self.nodes[0].backupwallet, sourcePath) + if __name__ == '__main__': WalletBackupTest().main() diff --git a/test/functional/zapwallettxes.py b/test/functional/zapwallettxes.py index 8cd622dc8e..0471f2afb3 100755 --- a/test/functional/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers +# Copyright (c) 2014-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the zapwallettxes functionality. diff --git a/test/functional/zmq_test.py b/test/functional/zmq_test.py index 382ef5bae2..86ccea4394 100755 --- a/test/functional/zmq_test.py +++ b/test/functional/zmq_test.py @@ -1,17 +1,38 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers +# Copyright (c) 2015-2017 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test the ZMQ API.""" +"""Test the ZMQ notification interface.""" import configparser import os import struct from test_framework.test_framework import BitcoinTestFramework, SkipTest +from test_framework.mininode import CTransaction from test_framework.util import (assert_equal, bytes_to_hex_str, hash256, ) +from io import BytesIO + +class ZMQSubscriber: + def __init__(self, socket, topic): + self.sequence = 0 + self.socket = socket + self.topic = topic + + import zmq + self.socket.setsockopt(zmq.SUBSCRIBE, self.topic) + + def receive(self): + topic, body, seq = self.socket.recv_multipart() + # Topic should match the subscriber topic. + assert_equal(topic, self.topic) + # Sequence should be incremental. + assert_equal(struct.unpack('<I', seq)[-1], self.sequence) + self.sequence += 1 + return body + class ZMQTest (BitcoinTestFramework): def set_test_params(self): @@ -24,26 +45,33 @@ class ZMQTest (BitcoinTestFramework): except ImportError: raise SkipTest("python3-zmq module not available.") - # Check that bitcoin has been built with ZMQ enabled + # Check that bitcoin has been built with ZMQ enabled. config = configparser.ConfigParser() if not self.options.configfile: - self.options.configfile = os.path.dirname(__file__) + "/../config.ini" + self.options.configfile = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config.ini")) config.read_file(open(self.options.configfile)) if not config["components"].getboolean("ENABLE_ZMQ"): raise SkipTest("bitcoind has not been built with zmq enabled.") - self.zmqContext = zmq.Context() - self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) - self.zmqSubSocket.set(zmq.RCVTIMEO, 60000) - self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashblock") - self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashtx") - self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"rawblock") - self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"rawtx") - ip_address = "tcp://127.0.0.1:28332" - self.zmqSubSocket.connect(ip_address) - self.extra_args = [['-zmqpubhashblock=%s' % ip_address, '-zmqpubhashtx=%s' % ip_address, - '-zmqpubrawblock=%s' % ip_address, '-zmqpubrawtx=%s' % ip_address], []] + # Initialize ZMQ context and socket. + # All messages are received in the same socket which means + # that this test fails if the publishing order changes. + # Note that the publishing order is not defined in the documentation and + # is subject to change. + address = "tcp://127.0.0.1:28332" + self.zmq_context = zmq.Context() + socket = self.zmq_context.socket(zmq.SUB) + socket.set(zmq.RCVTIMEO, 60000) + socket.connect(address) + + # Subscribe to all available topics. + self.hashblock = ZMQSubscriber(socket, b"hashblock") + self.hashtx = ZMQSubscriber(socket, b"hashtx") + self.rawblock = ZMQSubscriber(socket, b"rawblock") + self.rawtx = ZMQSubscriber(socket, b"rawtx") + + self.extra_args = [["-zmqpub%s=%s" % (sub.topic.decode(), address) for sub in [self.hashblock, self.hashtx, self.rawblock, self.rawtx]], []] self.add_nodes(self.num_nodes, self.extra_args) self.start_nodes() @@ -51,103 +79,48 @@ class ZMQTest (BitcoinTestFramework): try: self._zmq_test() finally: - # Destroy the zmq context - self.log.debug("Destroying zmq context") - self.zmqContext.destroy(linger=None) + # Destroy the ZMQ context. + self.log.debug("Destroying ZMQ context") + self.zmq_context.destroy(linger=None) def _zmq_test(self): - genhashes = self.nodes[0].generate(1) + num_blocks = 5 + self.log.info("Generate %(n)d blocks (and %(n)d coinbase txes)" % {"n": num_blocks}) + genhashes = self.nodes[0].generate(num_blocks) self.sync_all() - self.log.info("Wait for tx") - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"hashtx") - txhash = msg[1] - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, 0) # must be sequence 0 on hashtx - - # rawtx - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"rawtx") - body = msg[1] - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, 0) # must be sequence 0 on rawtx - - # Check that the rawtx hashes to the hashtx - assert_equal(hash256(body), txhash) - - self.log.info("Wait for block") - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"hashblock") - body = msg[1] - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, 0) # must be sequence 0 on hashblock - blkhash = bytes_to_hex_str(body) - assert_equal(genhashes[0], blkhash) # blockhash from generate must be equal to the hash received over zmq - - # rawblock - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"rawblock") - body = msg[1] - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, 0) #must be sequence 0 on rawblock - - # Check the hash of the rawblock's header matches generate - assert_equal(genhashes[0], bytes_to_hex_str(hash256(body[:80]))) - - self.log.info("Generate 10 blocks (and 10 coinbase txes)") - n = 10 - genhashes = self.nodes[1].generate(n) - self.sync_all() + for x in range(num_blocks): + # Should receive the coinbase txid. + txid = self.hashtx.receive() + + # Should receive the coinbase raw transaction. + hex = self.rawtx.receive() + tx = CTransaction() + tx.deserialize(BytesIO(hex)) + tx.calc_sha256() + assert_equal(tx.hash, bytes_to_hex_str(txid)) - zmqHashes = [] - zmqRawHashed = [] - blockcount = 0 - for x in range(n * 4): - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - body = msg[1] - if topic == b"hashblock": - zmqHashes.append(bytes_to_hex_str(body)) - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, blockcount + 1) - blockcount += 1 - if topic == b"rawblock": - zmqRawHashed.append(bytes_to_hex_str(hash256(body[:80]))) - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, blockcount) - - for x in range(n): - assert_equal(genhashes[x], zmqHashes[x]) # blockhash from generate must be equal to the hash received over zmq - assert_equal(genhashes[x], zmqRawHashed[x]) + # Should receive the generated block hash. + hash = bytes_to_hex_str(self.hashblock.receive()) + assert_equal(genhashes[x], hash) + # The block should only have the coinbase txid. + assert_equal([bytes_to_hex_str(txid)], self.nodes[1].getblock(hash)["tx"]) + + # Should receive the generated raw block. + block = self.rawblock.receive() + assert_equal(genhashes[x], bytes_to_hex_str(hash256(block[:80]))) self.log.info("Wait for tx from second node") - # test tx from a second node - hashRPC = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) + payment_txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) self.sync_all() - # now we should receive a zmq msg because the tx was broadcast - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"hashtx") - body = msg[1] - hashZMQ = bytes_to_hex_str(body) - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, blockcount + 1) - - msg = self.zmqSubSocket.recv_multipart() - topic = msg[0] - assert_equal(topic, b"rawtx") - body = msg[1] - hashedZMQ = bytes_to_hex_str(hash256(body)) - msgSequence = struct.unpack('<I', msg[-1])[-1] - assert_equal(msgSequence, blockcount+1) - assert_equal(hashRPC, hashZMQ) # txid from sendtoaddress must be equal to the hash received over zmq - assert_equal(hashRPC, hashedZMQ) + # Should receive the broadcasted txid. + txid = self.hashtx.receive() + assert_equal(payment_txid, bytes_to_hex_str(txid)) + + # Should receive the broadcasted raw transaction. + hex = self.rawtx.receive() + assert_equal(payment_txid, bytes_to_hex_str(hash256(hex))) if __name__ == '__main__': ZMQTest().main() diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py index ef34955d90..64e826ad0b 100755 --- a/test/util/bitcoin-util-test.py +++ b/test/util/bitcoin-util-test.py @@ -48,7 +48,7 @@ def main(): def bctester(testDir, input_basename, buildenv): """ Loads and parses the input file, runs all tests and reports results""" - input_filename = testDir + "/" + input_basename + input_filename = os.path.join(testDir, input_basename) raw_data = open(input_filename).read() input_data = json.loads(raw_data) @@ -77,7 +77,7 @@ def bctest(testDir, testObj, buildenv): are not as expected. Error is caught by bctester() and reported. """ # Get the exec names and arguments - execprog = buildenv["BUILDDIR"] + "/src/" + testObj['exec'] + buildenv["EXEEXT"] + execprog = os.path.join(buildenv["BUILDDIR"], "src", testObj["exec"] + buildenv["EXEEXT"]) execargs = testObj['args'] execrun = [execprog] + execargs @@ -85,24 +85,28 @@ def bctest(testDir, testObj, buildenv): stdinCfg = None inputData = None if "input" in testObj: - filename = testDir + "/" + testObj['input'] + filename = os.path.join(testDir, testObj["input"]) inputData = open(filename).read() stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) outputFn = None outputData = None + outputType = None if "output_cmp" in testObj: outputFn = testObj['output_cmp'] outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) try: - outputData = open(testDir + "/" + outputFn).read() + outputData = open(os.path.join(testDir, outputFn)).read() except: logging.error("Output file " + outputFn + " can not be opened") raise if not outputData: logging.error("Output data missing for " + outputFn) raise Exception + if not outputType: + logging.error("Output file %s does not have a file extension" % outputFn) + raise Exception # Run the test proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) diff --git a/test/util/data/tt-delin1-out.json b/test/util/data/tt-delin1-out.json index 0b3235dd4a..de647f98b6 100644 --- a/test/util/data/tt-delin1-out.json +++ b/test/util/data/tt-delin1-out.json @@ -14,7 +14,7 @@ "hex": "493046022100b4251ecd63778a3dde0155abe4cd162947620ae9ee45a874353551092325b116022100db307baf4ff3781ec520bd18f387948cedd15dc27bafe17c894b0fe6ffffcafa012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "752f7f69b915637dc1c2f7aed1466ad676f6f3e24cf922809705f664e97ab3c1", "vout": 1, @@ -23,7 +23,7 @@ "hex": "473044022079bd62ee09621a3be96b760c39e8ef78170101d46313923c6b07ae60a95c90670220238e51ea29fc70b04b65508450523caedbb11cb4dd5aa608c81487de798925ba0121027a759be8df971a6a04fafcb4f6babf75dc811c5cdaa0734cddbe9b942ce75b34" }, "sequence": 4294967295 - }, + }, { "txid": "b0ac9cca2e69cd02410e31b1f4402a25758e71abd1ab06c265ef9077dc05d0ed", "vout": 209, @@ -32,7 +32,7 @@ "hex": "48304502207722d6f9038673c86a1019b1c4de2d687ae246477cd4ca7002762be0299de385022100e594a11e3a313942595f7666dcf7078bcb14f1330f4206b95c917e7ec0e82fac012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "a135eafb595eaf4c1ea59ccb111cdc0eae1b2c979b226a1e5aa8b76fe2d628df", "vout": 0, @@ -41,7 +41,7 @@ "hex": "483045022100a63a4788027b79b65c6f9d9e054f68cf3b4eed19efd82a2d53f70dcbe64683390220526f243671425b2bd05745fcf2729361f985cfe84ea80c7cfc817b93d8134374012103a621f08be22d1bbdcbe4e527ee4927006aa555fc65e2aafa767d4ea2fe9dfa52" }, "sequence": 4294967295 - }, + }, { "txid": "a5d6bf53ba21140b8a4d554feb00fe8bb9a62430ff9e4624aa2f58a120232aae", "vout": 1, @@ -50,7 +50,7 @@ "hex": "493046022100b200ac6db16842f76dab9abe807ce423c992805879bc50abd46ed8275a59d9cf022100c0d518e85dd345b3c29dd4dc47b9a420d3ce817b18720e94966d2fe23413a408012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "1b299cf14f1a22e81ea56d71b7affbd7cf386807bf2b4d4b79a18a54125accb3", "vout": 0, @@ -59,7 +59,7 @@ "hex": "483045022100ededc441c3103a6f2bd6cab7639421af0f6ec5e60503bce1e603cf34f00aee1c02205cb75f3f519a13fb348783b21db3085cb5ec7552c59e394fdbc3e1feea43f967012103a621f08be22d1bbdcbe4e527ee4927006aa555fc65e2aafa767d4ea2fe9dfa52" }, "sequence": 4294967295 - }, + }, { "txid": "071df1cdcb3f0070f9d6af7b0274f02d0be2324a274727cfd288383167531485", "vout": 21, @@ -68,7 +68,7 @@ "hex": "483045022100d9eed5413d2a4b4b98625aa6e3169edc4fb4663e7862316d69224454e70cd8ca022061e506521d5ced51dd0ea36496e75904d756a4c4f9fb111568555075d5f68d9a012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "b012e500eb7adf7a13ed332dd6ece849f94f7a62bb3eac5babab356d1fc19282", "vout": 9, @@ -77,7 +77,7 @@ "hex": "48304502207e84b27139c4c19c828cb1e30c349bba88e4d9b59be97286960793b5ddc0a2af0221008cdc7a951e7f31c20953ed5635fbabf228e80b7047f32faaa0313e7693005177012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "58840fee9c833f2f2d40575842f30f4b8d2553094d06ad88b03d06869acf3d88", "vout": 30, @@ -86,7 +86,7 @@ "hex": "4730440220426540dfed9c4ab5812e5f06df705b8bcf307dd7d20f7fa6512298b2a6314f420220064055096e3ca62f6c7352c66a5447767c53f946acdf35025ab3807ddb2fa404012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "e69f9cd16946e570a665245354428a3f507ea69f4568b581e4af98edb3db9766", "vout": 114, @@ -95,7 +95,7 @@ "hex": "47304402200a5e673996f2fc88e21cc8613611f08a650bc0370338803591d85d0ec5663764022040b6664a0d1ec83a7f01975b8fde5232992b8ca58bf48af6725d2f92a936ab2e012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "595d1257f654ed2cbe5a65421e8aefd2b4d70b5b6c89a03f1d7e518221fc3f02", "vout": 103, @@ -104,7 +104,7 @@ "hex": "493046022100d93b30219c5735f673be5c3b4688366d96f545561c74cb62c6958c00f6960806022100ec8200adcb028f2184fa2a4f6faac7f8bb57cb4503bb7584ac11051fece31b3d012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "06fc818f9555a261248ecd7aad0993eafb5a82ceb2b5c87c3ddfb06671c7f816", "vout": 1, @@ -113,7 +113,7 @@ "hex": "483045022100a13934e68d3f5b22b130c4cb33f4da468cffc52323a47fbfbe06b64858162246022047081e0a70ff770e64a2e2d31e5d520d9102268b57a47009a72fe73ec766901801210234b9d9413f247bb78cd3293b7b65a2c38018ba5621ea9ee737f3a6a3523fb4cd" }, "sequence": 4294967295 - }, + }, { "txid": "fb416c8155d6bb1d43f9395466ca90a638a7c2dd3ff617aadf3a7ac8f3967b19", "vout": 0, @@ -122,7 +122,7 @@ "hex": "49304602210097f1f35d5bdc1a3a60390a1b015b8e7c4f916aa3847aafd969e04975e15bbe70022100a9052eb25517d481f1fda1b129eb1b534da50ea1a51f3ee012dca3601c11b86a0121027a759be8df971a6a04fafcb4f6babf75dc811c5cdaa0734cddbe9b942ce75b34" }, "sequence": 4294967295 - }, + }, { "txid": "3940b9683bd6104ad24c978e640ba4095993cafdb27d2ed91baa27ee61a2d920", "vout": 221, @@ -131,7 +131,7 @@ "hex": "483045022012b3138c591bf7154b6fef457f2c4a3c7162225003788ac0024a99355865ff13022100b71b125ae1ffb2e1d1571f580cd3ebc8cd049a2d7a8a41f138ba94aeb982106f012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "711b5714d3b5136147c02194cd95bde94a4648c4263ca6f972d86cd1d579f150", "vout": 1, @@ -140,7 +140,7 @@ "hex": "483045022100f834ccc8b22ee72712a3e5e6ef4acb8b2fb791b5385b70e2cd4332674d6667f4022024fbda0a997e0c253503f217501f508a4d56edce2c813ecdd9ad796dbeba907401210234b9d9413f247bb78cd3293b7b65a2c38018ba5621ea9ee737f3a6a3523fb4cd" }, "sequence": 4294967295 - }, + }, { "txid": "6364b5c5efe018430789e7fb4e338209546cae5d9c5f5e300aac68155d861b55", "vout": 27, @@ -149,7 +149,7 @@ "hex": "48304502203b2fd1e39ae0e469d7a15768f262661b0de41470daf0fe8c4fd0c26542a0870002210081c57e331f9a2d214457d953e3542904727ee412c63028113635d7224da3dccc012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "0bb57f6e38012c86d4c5a28c904f2675082859147921a707d48961015a3e5057", "vout": 1095, @@ -158,7 +158,7 @@ "hex": "48304502206947a9c54f0664ece4430fd4ae999891dc50bb6126bc36b6a15a3189f29d25e9022100a86cfc4e2fdd9e39a20e305cfd1b76509c67b3e313e0f118229105caa0e823c9012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "9b34274814a2540bb062107117f8f3e75ef85d953e9372d8261a3e9dfbc1163f", "vout": 37, @@ -167,7 +167,7 @@ "hex": "483045022100c7128fe10b2d38744ae8177776054c29fc8ec13f07207723e70766ab7164847402201d2cf09009b9596de74c0183d1ab832e5edddb7a9965880bb400097e850850f8012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "b86b5cc0d8a7374d94e277850b0a249cb26a7b42ddf014f28a49b8859da64241", "vout": 20, @@ -176,7 +176,7 @@ "hex": "48304502203b89a71628a28cc3703d170ca3be77786cff6b867e38a18b719705f8a326578f022100b2a9879e1acf621faa6466c207746a7f3eb4c8514c1482969aba3f2a957f1321012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "3d0a2353eeec44d3c10aed259038db321912122cd4150048f7bfa4c0ecfee236", "vout": 242, @@ -200,7 +200,7 @@ "1E7SGgAZFCHDnVZLuRViX3gUmxpMfdvd2o" ] } - }, + }, { "value": 0.01000001, "n": 1, diff --git a/test/util/data/tt-delout1-out.json b/test/util/data/tt-delout1-out.json index 5b69d0cd86..067ffe74e7 100644 --- a/test/util/data/tt-delout1-out.json +++ b/test/util/data/tt-delout1-out.json @@ -14,7 +14,7 @@ "hex": "493046022100b4251ecd63778a3dde0155abe4cd162947620ae9ee45a874353551092325b116022100db307baf4ff3781ec520bd18f387948cedd15dc27bafe17c894b0fe6ffffcafa012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "a72ec96bd0d022d1b0c2f9078cdd46b3725b8eecdd001e17b21e3ababad14ecb", "vout": 0, @@ -23,7 +23,7 @@ "hex": "493046022100a9b617843b68c284715d3e02fd120479cd0d96a6c43bf01e697fb0a460a21a3a022100ba0a12fbe8b993d4e7911fa3467615765dbe421ddf5c51b57a9c1ee19dcc00ba012103e633b4fa4ceb705c2da712390767199be8ef2448b3095dc01652e11b2b751505" }, "sequence": 4294967295 - }, + }, { "txid": "752f7f69b915637dc1c2f7aed1466ad676f6f3e24cf922809705f664e97ab3c1", "vout": 1, @@ -32,7 +32,7 @@ "hex": "473044022079bd62ee09621a3be96b760c39e8ef78170101d46313923c6b07ae60a95c90670220238e51ea29fc70b04b65508450523caedbb11cb4dd5aa608c81487de798925ba0121027a759be8df971a6a04fafcb4f6babf75dc811c5cdaa0734cddbe9b942ce75b34" }, "sequence": 4294967295 - }, + }, { "txid": "b0ac9cca2e69cd02410e31b1f4402a25758e71abd1ab06c265ef9077dc05d0ed", "vout": 209, @@ -41,7 +41,7 @@ "hex": "48304502207722d6f9038673c86a1019b1c4de2d687ae246477cd4ca7002762be0299de385022100e594a11e3a313942595f7666dcf7078bcb14f1330f4206b95c917e7ec0e82fac012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "a135eafb595eaf4c1ea59ccb111cdc0eae1b2c979b226a1e5aa8b76fe2d628df", "vout": 0, @@ -50,7 +50,7 @@ "hex": "483045022100a63a4788027b79b65c6f9d9e054f68cf3b4eed19efd82a2d53f70dcbe64683390220526f243671425b2bd05745fcf2729361f985cfe84ea80c7cfc817b93d8134374012103a621f08be22d1bbdcbe4e527ee4927006aa555fc65e2aafa767d4ea2fe9dfa52" }, "sequence": 4294967295 - }, + }, { "txid": "a5d6bf53ba21140b8a4d554feb00fe8bb9a62430ff9e4624aa2f58a120232aae", "vout": 1, @@ -59,7 +59,7 @@ "hex": "493046022100b200ac6db16842f76dab9abe807ce423c992805879bc50abd46ed8275a59d9cf022100c0d518e85dd345b3c29dd4dc47b9a420d3ce817b18720e94966d2fe23413a408012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "1b299cf14f1a22e81ea56d71b7affbd7cf386807bf2b4d4b79a18a54125accb3", "vout": 0, @@ -68,7 +68,7 @@ "hex": "483045022100ededc441c3103a6f2bd6cab7639421af0f6ec5e60503bce1e603cf34f00aee1c02205cb75f3f519a13fb348783b21db3085cb5ec7552c59e394fdbc3e1feea43f967012103a621f08be22d1bbdcbe4e527ee4927006aa555fc65e2aafa767d4ea2fe9dfa52" }, "sequence": 4294967295 - }, + }, { "txid": "071df1cdcb3f0070f9d6af7b0274f02d0be2324a274727cfd288383167531485", "vout": 21, @@ -77,7 +77,7 @@ "hex": "483045022100d9eed5413d2a4b4b98625aa6e3169edc4fb4663e7862316d69224454e70cd8ca022061e506521d5ced51dd0ea36496e75904d756a4c4f9fb111568555075d5f68d9a012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "b012e500eb7adf7a13ed332dd6ece849f94f7a62bb3eac5babab356d1fc19282", "vout": 9, @@ -86,7 +86,7 @@ "hex": "48304502207e84b27139c4c19c828cb1e30c349bba88e4d9b59be97286960793b5ddc0a2af0221008cdc7a951e7f31c20953ed5635fbabf228e80b7047f32faaa0313e7693005177012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "58840fee9c833f2f2d40575842f30f4b8d2553094d06ad88b03d06869acf3d88", "vout": 30, @@ -95,7 +95,7 @@ "hex": "4730440220426540dfed9c4ab5812e5f06df705b8bcf307dd7d20f7fa6512298b2a6314f420220064055096e3ca62f6c7352c66a5447767c53f946acdf35025ab3807ddb2fa404012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "e69f9cd16946e570a665245354428a3f507ea69f4568b581e4af98edb3db9766", "vout": 114, @@ -104,7 +104,7 @@ "hex": "47304402200a5e673996f2fc88e21cc8613611f08a650bc0370338803591d85d0ec5663764022040b6664a0d1ec83a7f01975b8fde5232992b8ca58bf48af6725d2f92a936ab2e012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "595d1257f654ed2cbe5a65421e8aefd2b4d70b5b6c89a03f1d7e518221fc3f02", "vout": 103, @@ -113,7 +113,7 @@ "hex": "493046022100d93b30219c5735f673be5c3b4688366d96f545561c74cb62c6958c00f6960806022100ec8200adcb028f2184fa2a4f6faac7f8bb57cb4503bb7584ac11051fece31b3d012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "06fc818f9555a261248ecd7aad0993eafb5a82ceb2b5c87c3ddfb06671c7f816", "vout": 1, @@ -122,7 +122,7 @@ "hex": "483045022100a13934e68d3f5b22b130c4cb33f4da468cffc52323a47fbfbe06b64858162246022047081e0a70ff770e64a2e2d31e5d520d9102268b57a47009a72fe73ec766901801210234b9d9413f247bb78cd3293b7b65a2c38018ba5621ea9ee737f3a6a3523fb4cd" }, "sequence": 4294967295 - }, + }, { "txid": "fb416c8155d6bb1d43f9395466ca90a638a7c2dd3ff617aadf3a7ac8f3967b19", "vout": 0, @@ -131,7 +131,7 @@ "hex": "49304602210097f1f35d5bdc1a3a60390a1b015b8e7c4f916aa3847aafd969e04975e15bbe70022100a9052eb25517d481f1fda1b129eb1b534da50ea1a51f3ee012dca3601c11b86a0121027a759be8df971a6a04fafcb4f6babf75dc811c5cdaa0734cddbe9b942ce75b34" }, "sequence": 4294967295 - }, + }, { "txid": "3940b9683bd6104ad24c978e640ba4095993cafdb27d2ed91baa27ee61a2d920", "vout": 221, @@ -140,7 +140,7 @@ "hex": "483045022012b3138c591bf7154b6fef457f2c4a3c7162225003788ac0024a99355865ff13022100b71b125ae1ffb2e1d1571f580cd3ebc8cd049a2d7a8a41f138ba94aeb982106f012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "711b5714d3b5136147c02194cd95bde94a4648c4263ca6f972d86cd1d579f150", "vout": 1, @@ -149,7 +149,7 @@ "hex": "483045022100f834ccc8b22ee72712a3e5e6ef4acb8b2fb791b5385b70e2cd4332674d6667f4022024fbda0a997e0c253503f217501f508a4d56edce2c813ecdd9ad796dbeba907401210234b9d9413f247bb78cd3293b7b65a2c38018ba5621ea9ee737f3a6a3523fb4cd" }, "sequence": 4294967295 - }, + }, { "txid": "6364b5c5efe018430789e7fb4e338209546cae5d9c5f5e300aac68155d861b55", "vout": 27, @@ -158,7 +158,7 @@ "hex": "48304502203b2fd1e39ae0e469d7a15768f262661b0de41470daf0fe8c4fd0c26542a0870002210081c57e331f9a2d214457d953e3542904727ee412c63028113635d7224da3dccc012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "0bb57f6e38012c86d4c5a28c904f2675082859147921a707d48961015a3e5057", "vout": 1095, @@ -167,7 +167,7 @@ "hex": "48304502206947a9c54f0664ece4430fd4ae999891dc50bb6126bc36b6a15a3189f29d25e9022100a86cfc4e2fdd9e39a20e305cfd1b76509c67b3e313e0f118229105caa0e823c9012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "9b34274814a2540bb062107117f8f3e75ef85d953e9372d8261a3e9dfbc1163f", "vout": 37, @@ -176,7 +176,7 @@ "hex": "483045022100c7128fe10b2d38744ae8177776054c29fc8ec13f07207723e70766ab7164847402201d2cf09009b9596de74c0183d1ab832e5edddb7a9965880bb400097e850850f8012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "b86b5cc0d8a7374d94e277850b0a249cb26a7b42ddf014f28a49b8859da64241", "vout": 20, @@ -185,7 +185,7 @@ "hex": "48304502203b89a71628a28cc3703d170ca3be77786cff6b867e38a18b719705f8a326578f022100b2a9879e1acf621faa6466c207746a7f3eb4c8514c1482969aba3f2a957f1321012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "3d0a2353eeec44d3c10aed259038db321912122cd4150048f7bfa4c0ecfee236", "vout": 242, diff --git a/test/util/data/tt-locktime317000-out.json b/test/util/data/tt-locktime317000-out.json index cf1ebcdf38..af7903d1dd 100644 --- a/test/util/data/tt-locktime317000-out.json +++ b/test/util/data/tt-locktime317000-out.json @@ -14,7 +14,7 @@ "hex": "493046022100b4251ecd63778a3dde0155abe4cd162947620ae9ee45a874353551092325b116022100db307baf4ff3781ec520bd18f387948cedd15dc27bafe17c894b0fe6ffffcafa012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "a72ec96bd0d022d1b0c2f9078cdd46b3725b8eecdd001e17b21e3ababad14ecb", "vout": 0, @@ -23,7 +23,7 @@ "hex": "493046022100a9b617843b68c284715d3e02fd120479cd0d96a6c43bf01e697fb0a460a21a3a022100ba0a12fbe8b993d4e7911fa3467615765dbe421ddf5c51b57a9c1ee19dcc00ba012103e633b4fa4ceb705c2da712390767199be8ef2448b3095dc01652e11b2b751505" }, "sequence": 4294967295 - }, + }, { "txid": "752f7f69b915637dc1c2f7aed1466ad676f6f3e24cf922809705f664e97ab3c1", "vout": 1, @@ -32,7 +32,7 @@ "hex": "473044022079bd62ee09621a3be96b760c39e8ef78170101d46313923c6b07ae60a95c90670220238e51ea29fc70b04b65508450523caedbb11cb4dd5aa608c81487de798925ba0121027a759be8df971a6a04fafcb4f6babf75dc811c5cdaa0734cddbe9b942ce75b34" }, "sequence": 4294967295 - }, + }, { "txid": "b0ac9cca2e69cd02410e31b1f4402a25758e71abd1ab06c265ef9077dc05d0ed", "vout": 209, @@ -41,7 +41,7 @@ "hex": "48304502207722d6f9038673c86a1019b1c4de2d687ae246477cd4ca7002762be0299de385022100e594a11e3a313942595f7666dcf7078bcb14f1330f4206b95c917e7ec0e82fac012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "a135eafb595eaf4c1ea59ccb111cdc0eae1b2c979b226a1e5aa8b76fe2d628df", "vout": 0, @@ -50,7 +50,7 @@ "hex": "483045022100a63a4788027b79b65c6f9d9e054f68cf3b4eed19efd82a2d53f70dcbe64683390220526f243671425b2bd05745fcf2729361f985cfe84ea80c7cfc817b93d8134374012103a621f08be22d1bbdcbe4e527ee4927006aa555fc65e2aafa767d4ea2fe9dfa52" }, "sequence": 4294967295 - }, + }, { "txid": "a5d6bf53ba21140b8a4d554feb00fe8bb9a62430ff9e4624aa2f58a120232aae", "vout": 1, @@ -59,7 +59,7 @@ "hex": "493046022100b200ac6db16842f76dab9abe807ce423c992805879bc50abd46ed8275a59d9cf022100c0d518e85dd345b3c29dd4dc47b9a420d3ce817b18720e94966d2fe23413a408012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "1b299cf14f1a22e81ea56d71b7affbd7cf386807bf2b4d4b79a18a54125accb3", "vout": 0, @@ -68,7 +68,7 @@ "hex": "483045022100ededc441c3103a6f2bd6cab7639421af0f6ec5e60503bce1e603cf34f00aee1c02205cb75f3f519a13fb348783b21db3085cb5ec7552c59e394fdbc3e1feea43f967012103a621f08be22d1bbdcbe4e527ee4927006aa555fc65e2aafa767d4ea2fe9dfa52" }, "sequence": 4294967295 - }, + }, { "txid": "071df1cdcb3f0070f9d6af7b0274f02d0be2324a274727cfd288383167531485", "vout": 21, @@ -77,7 +77,7 @@ "hex": "483045022100d9eed5413d2a4b4b98625aa6e3169edc4fb4663e7862316d69224454e70cd8ca022061e506521d5ced51dd0ea36496e75904d756a4c4f9fb111568555075d5f68d9a012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "b012e500eb7adf7a13ed332dd6ece849f94f7a62bb3eac5babab356d1fc19282", "vout": 9, @@ -86,7 +86,7 @@ "hex": "48304502207e84b27139c4c19c828cb1e30c349bba88e4d9b59be97286960793b5ddc0a2af0221008cdc7a951e7f31c20953ed5635fbabf228e80b7047f32faaa0313e7693005177012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "58840fee9c833f2f2d40575842f30f4b8d2553094d06ad88b03d06869acf3d88", "vout": 30, @@ -95,7 +95,7 @@ "hex": "4730440220426540dfed9c4ab5812e5f06df705b8bcf307dd7d20f7fa6512298b2a6314f420220064055096e3ca62f6c7352c66a5447767c53f946acdf35025ab3807ddb2fa404012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "e69f9cd16946e570a665245354428a3f507ea69f4568b581e4af98edb3db9766", "vout": 114, @@ -104,7 +104,7 @@ "hex": "47304402200a5e673996f2fc88e21cc8613611f08a650bc0370338803591d85d0ec5663764022040b6664a0d1ec83a7f01975b8fde5232992b8ca58bf48af6725d2f92a936ab2e012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "595d1257f654ed2cbe5a65421e8aefd2b4d70b5b6c89a03f1d7e518221fc3f02", "vout": 103, @@ -113,7 +113,7 @@ "hex": "493046022100d93b30219c5735f673be5c3b4688366d96f545561c74cb62c6958c00f6960806022100ec8200adcb028f2184fa2a4f6faac7f8bb57cb4503bb7584ac11051fece31b3d012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "06fc818f9555a261248ecd7aad0993eafb5a82ceb2b5c87c3ddfb06671c7f816", "vout": 1, @@ -122,7 +122,7 @@ "hex": "483045022100a13934e68d3f5b22b130c4cb33f4da468cffc52323a47fbfbe06b64858162246022047081e0a70ff770e64a2e2d31e5d520d9102268b57a47009a72fe73ec766901801210234b9d9413f247bb78cd3293b7b65a2c38018ba5621ea9ee737f3a6a3523fb4cd" }, "sequence": 4294967295 - }, + }, { "txid": "fb416c8155d6bb1d43f9395466ca90a638a7c2dd3ff617aadf3a7ac8f3967b19", "vout": 0, @@ -131,7 +131,7 @@ "hex": "49304602210097f1f35d5bdc1a3a60390a1b015b8e7c4f916aa3847aafd969e04975e15bbe70022100a9052eb25517d481f1fda1b129eb1b534da50ea1a51f3ee012dca3601c11b86a0121027a759be8df971a6a04fafcb4f6babf75dc811c5cdaa0734cddbe9b942ce75b34" }, "sequence": 4294967295 - }, + }, { "txid": "3940b9683bd6104ad24c978e640ba4095993cafdb27d2ed91baa27ee61a2d920", "vout": 221, @@ -140,7 +140,7 @@ "hex": "483045022012b3138c591bf7154b6fef457f2c4a3c7162225003788ac0024a99355865ff13022100b71b125ae1ffb2e1d1571f580cd3ebc8cd049a2d7a8a41f138ba94aeb982106f012103091137f3ef23f4acfc19a5953a68b2074fae942ad3563ef28c33b0cac9a93adc" }, "sequence": 4294967295 - }, + }, { "txid": "711b5714d3b5136147c02194cd95bde94a4648c4263ca6f972d86cd1d579f150", "vout": 1, @@ -149,7 +149,7 @@ "hex": "483045022100f834ccc8b22ee72712a3e5e6ef4acb8b2fb791b5385b70e2cd4332674d6667f4022024fbda0a997e0c253503f217501f508a4d56edce2c813ecdd9ad796dbeba907401210234b9d9413f247bb78cd3293b7b65a2c38018ba5621ea9ee737f3a6a3523fb4cd" }, "sequence": 4294967295 - }, + }, { "txid": "6364b5c5efe018430789e7fb4e338209546cae5d9c5f5e300aac68155d861b55", "vout": 27, @@ -158,7 +158,7 @@ "hex": "48304502203b2fd1e39ae0e469d7a15768f262661b0de41470daf0fe8c4fd0c26542a0870002210081c57e331f9a2d214457d953e3542904727ee412c63028113635d7224da3dccc012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "0bb57f6e38012c86d4c5a28c904f2675082859147921a707d48961015a3e5057", "vout": 1095, @@ -167,7 +167,7 @@ "hex": "48304502206947a9c54f0664ece4430fd4ae999891dc50bb6126bc36b6a15a3189f29d25e9022100a86cfc4e2fdd9e39a20e305cfd1b76509c67b3e313e0f118229105caa0e823c9012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "9b34274814a2540bb062107117f8f3e75ef85d953e9372d8261a3e9dfbc1163f", "vout": 37, @@ -176,7 +176,7 @@ "hex": "483045022100c7128fe10b2d38744ae8177776054c29fc8ec13f07207723e70766ab7164847402201d2cf09009b9596de74c0183d1ab832e5edddb7a9965880bb400097e850850f8012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "b86b5cc0d8a7374d94e277850b0a249cb26a7b42ddf014f28a49b8859da64241", "vout": 20, @@ -185,7 +185,7 @@ "hex": "48304502203b89a71628a28cc3703d170ca3be77786cff6b867e38a18b719705f8a326578f022100b2a9879e1acf621faa6466c207746a7f3eb4c8514c1482969aba3f2a957f1321012103f1575d6124ac78be398c25b31146d08313c6072d23a4d7df5ac6a9f87346c64c" }, "sequence": 4294967295 - }, + }, { "txid": "3d0a2353eeec44d3c10aed259038db321912122cd4150048f7bfa4c0ecfee236", "vout": 242, @@ -209,7 +209,7 @@ "1E7SGgAZFCHDnVZLuRViX3gUmxpMfdvd2o" ] } - }, + }, { "value": 0.01000001, "n": 1, diff --git a/test/util/data/txcreate1.json b/test/util/data/txcreate1.json index edb091f946..83a86649e0 100644 --- a/test/util/data/txcreate1.json +++ b/test/util/data/txcreate1.json @@ -14,7 +14,7 @@ "hex": "" }, "sequence": 4294967295 - }, + }, { "txid": "bf829c6bcf84579331337659d31f89dfd138f7f7785802d5501c92333145ca7c", "vout": 18, @@ -23,7 +23,7 @@ "hex": "" }, "sequence": 4294967295 - }, + }, { "txid": "22a6f904655d53ae2ff70e701a0bbd90aa3975c0f40bfc6cc996a9049e31cdfc", "vout": 1, @@ -47,7 +47,7 @@ "13tuJJDR2RgArmgfv6JScSdreahzgc4T6o" ] } - }, + }, { "value": 4.00000000, "n": 1, diff --git a/test/util/data/txcreatedata1.json b/test/util/data/txcreatedata1.json index e66a6bb9a5..15a4246ae5 100644 --- a/test/util/data/txcreatedata1.json +++ b/test/util/data/txcreatedata1.json @@ -29,7 +29,7 @@ "13tuJJDR2RgArmgfv6JScSdreahzgc4T6o" ] } - }, + }, { "value": 4.00000000, "n": 1, diff --git a/test/util/data/txcreatedata2.json b/test/util/data/txcreatedata2.json index 0f8edcafdd..cb93c27971 100644 --- a/test/util/data/txcreatedata2.json +++ b/test/util/data/txcreatedata2.json @@ -29,7 +29,7 @@ "13tuJJDR2RgArmgfv6JScSdreahzgc4T6o" ] } - }, + }, { "value": 0.00000000, "n": 1, diff --git a/test/util/data/txcreatedata_seq1.json b/test/util/data/txcreatedata_seq1.json index 771ff1bb10..dea48ba373 100644 --- a/test/util/data/txcreatedata_seq1.json +++ b/test/util/data/txcreatedata_seq1.json @@ -14,7 +14,7 @@ "hex": "" }, "sequence": 4294967293 - }, + }, { "txid": "5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f", "vout": 0, diff --git a/test/util/data/txcreatemultisig1.json b/test/util/data/txcreatemultisig1.json index 7c814dad83..72e20c8691 100644 --- a/test/util/data/txcreatemultisig1.json +++ b/test/util/data/txcreatemultisig1.json @@ -17,8 +17,8 @@ "reqSigs": 2, "type": "multisig", "addresses": [ - "1FoG2386FG2tAJS9acMuiDsKy67aGg9MKz", - "1FXtz9KU8JNmQDyHdiEm5HDiALuP3zdHvV", + "1FoG2386FG2tAJS9acMuiDsKy67aGg9MKz", + "1FXtz9KU8JNmQDyHdiEm5HDiALuP3zdHvV", "14LuavcBbXZYJ6Tsz3cAUQj9SuQoL2xCQX" ] } |