diff options
306 files changed, 8659 insertions, 5407 deletions
diff --git a/.travis.yml b/.travis.yml index a479e46f44..fcf4b082f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ env: # bitcoind - HOST=x86_64-unknown-linux-gnu PACKAGES="bc python3-zmq" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" # No wallet - - HOST=x86_64-unknown-linux-gnu PACKAGES="python3 xvfb" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" + - HOST=x86_64-unknown-linux-gnu PACKAGES="python3" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" # Cross-Mac - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" OSX_SDK=10.11 GOAL="deploy" @@ -43,14 +43,13 @@ install: - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi before_script: + - if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then contrib/devtools/commit-script-check.sh $TRAVIS_COMMIT_RANGE; fi - unset CC; unset CXX - if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi - mkdir -p depends/SDKs depends/sdk-sources - if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS - # Start xvfb if needed, as documented at https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI - - if [ "$RUN_TESTS" = "true" -a "${DEP_OPTS#*NO_QT=1}" = "$DEP_OPTS" ]; then export DISPLAY=:99.0; /sbin/start-stop-daemon --start --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac; fi script: - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then while read LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys; fi - if [ "$CHECK_DOC" = 1 -a "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_PULL_REQUEST" = "false" ]; then git fetch --unshallow; fi @@ -68,7 +67,7 @@ script: - ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib - - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS check VERBOSE=1; fi + - if [ "$RUN_TESTS" = "true" ]; then travis_wait 30 make $MAKEJOBS check VERBOSE=1; fi - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then extended="--extended --exclude pruning"; fi - if [ "$RUN_TESTS" = "true" ]; then test/functional/test_runner.py --coverage --quiet ${extended}; fi after_script: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0766d89f55..f5d63517b1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,8 +37,8 @@ fixes or code moves with actual code changes. Commit messages should be verbose by default consisting of a short subject line (50 chars max), a blank line and detailed explanatory text as separate -paragraph(s); unless the title alone is self-explanatory (like "Corrected typo -in init.cpp") then a single title line is sufficient. Commit messages should be +paragraph(s), unless the title alone is self-explanatory (like "Corrected typo +in init.cpp") in which case a single title line is sufficient. Commit messages should be helpful to people reading your code in the future, so explain the reasoning for your decisions. Further explanation [here](http://chris.beams.io/posts/git-commit/). @@ -225,6 +225,37 @@ discussed extensively on the mailing list and IRC, be accompanied by a widely discussed BIP and have a generally widely perceived technical consensus of being a worthwhile change based on the judgement of the maintainers. +### Finding Reviewers + +As most reviewers are themselves developers with their own projects, the review +process can be quite lengthy, and some amount of patience is required. If you find +that you've been waiting for a pull request to be given attention for several +months, there may be a number of reasons for this, some of which you can do something +about: + + - It may be because of a feature freeze due to an upcoming release. During this time, + only bug fixes are taken into consideration. If your pull request is a new feature, + it will not be prioritized until the release is over. Wait for release. + - It may be because the changes you are suggesting do not appeal to people. Rather than + nits and critique, which require effort and means they care enough to spend time on your + contribution, thundering silence is a good sign of widespread (mild) dislike of a given change + (because people don't assume *others* won't actually like the proposal). Don't take + that personally, though! Instead, take another critical look at what you are suggesting + and see if it: changes too much, is too broad, doesn't adhere to the + [developer notes](doc/developer-notes.md), is dangerous or insecure, is messily written, etc. + Identify and address any of the issues you find. Then ask e.g. on IRC if someone could give + their opinion on the concept itself. + - It may be because your code is too complex for all but a few people. And those people + may not have realized your pull request even exists. A great way to find people who + are qualified and care about the code you are touching is the + [Git Blame feature](https://help.github.com/articles/tracing-changes-in-a-file/). Simply + find the person touching the code you are touching before you and see if you can find + them and give them a nudge. Don't be incessant about the nudging though. + - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request + a look. If you think you've been waiting an unreasonably long amount of time (month+) for + no particular reason (few lines changed, etc), this is totally fine. Try to return the favor + when someone else is asking for feedback on their code, and universe balances out. + Release Policy -------------- diff --git a/Makefile.am b/Makefile.am index e82704dbdf..3a56eea0c0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -76,9 +76,6 @@ $(BITCOIN_WIN_INSTALLER): all-recursive echo error: could not build $@ @echo built $@ -$(if $(findstring src/,$(MAKECMDGOALS)),$(MAKECMDGOALS), none): FORCE - $(MAKE) -C src $(patsubst src/%,%,$@) - $(OSX_APP)/Contents/PkgInfo: $(MKDIR_P) $(@D) @echo "APPL????" > $@ diff --git a/configure.ac b/configure.ac index 1e1ed49f91..ca66216554 100644 --- a/configure.ac +++ b/configure.ac @@ -235,6 +235,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wself-assign],[CXXFLAGS="$CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[CXXFLAGS="$CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdeprecated-register],[CXXFLAGS="$CXXFLAGS -Wno-deprecated-register"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[CXXFLAGS="$CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) fi CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" @@ -549,6 +550,8 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, #include <byteswap.h> #endif]) +AC_CHECK_DECLS([__builtin_clz, __builtin_clzl, __builtin_clzll]) + dnl Check for MSG_NOSIGNAL AC_MSG_CHECKING(for MSG_NOSIGNAL) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/socket.h>]], @@ -676,6 +679,10 @@ AX_BOOST_PROGRAM_OPTIONS AX_BOOST_THREAD AX_BOOST_CHRONO +dnl Boost 1.56 through 1.62 allow using std::atomic instead of its own atomic +dnl counter implementations. In 1.63 and later the std::atomic approach is default. +m4_pattern_allow(DBOOST_AC_USE_STD_ATOMIC) dnl otherwise it's treated like a macro +BOOST_CPPFLAGS="-DBOOST_SP_USE_STD_ATOMIC -DBOOST_AC_USE_STD_ATOMIC $BOOST_CPPFLAGS" if test x$use_reduce_exports = xyes; then AC_MSG_CHECKING([for working boost reduced exports]) diff --git a/contrib/debian/changelog b/contrib/debian/changelog index 110bfe03ef..33dab9b638 100644 --- a/contrib/debian/changelog +++ b/contrib/debian/changelog @@ -1,3 +1,122 @@ +bitcoin (0.14.1-trusty4) trusty; urgency=medium + + * Re-enable UPnP support. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Fri, 05 May 2017 13:28:00 -0400 + +bitcoin (0.14.1-trusty3) trusty; urgency=medium + + * Build with qt5 if we are on a non-Ubuntu (ie non-Unity) distro. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Thu, 04 May 2017 17:13:00 -0400 + +bitcoin (0.14.1-trusty2) trusty; urgency=medium + + * Bump minimum boost version in deps. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Thu, 04 May 2017 17:12:00 -0400 + +bitcoin (0.14.1-trusty1) trusty; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Sat, 22 Apr 2017 17:10:00 -0400 + +bitcoin (0.14.0-trusty1) trusty; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Wed, 08 Mar 2017 10:30:00 -0500 + +bitcoin (0.13.2-trusty1) trusty; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Thu, 05 Jan 2017 09:59:00 -0500 + +bitcoin (0.13.1-trusty2) trusty; urgency=medium + + * Revert to Qt4, due to https://github.com/bitcoin/bitcoin/issues/9038 + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Mon, 31 Oct 2016 11:16:00 -0400 + +bitcoin (0.13.1-trusty1) trusty; urgency=medium + + * New upstream release. + * Backport updated bitcoin-qt.desktop from upstream master + * Add zmq dependency + * Switch to Qt5 (breaks precise, but that was already broken by C++11) + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Thu, 27 Oct 2016 17:32:00 -0400 + +bitcoin (0.13.0-trusty1) trusty; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Sun, 04 Sep 2016 22:09:00 -0400 + +bitcoin (0.12.1-trusty1) trusty; urgency=medium + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Mon, 18 Apr 2016 14:26:00 -0700 + +bitcoin (0.12.0-trusty6) trusty; urgency=medium + + * Fix program-options dep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Fri, 25 Mar 2016 21:41:00 -0700 + +bitcoin (0.12.0-trusty5) trusty; urgency=medium + + * Test explicit --with-gui + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 23 Feb 2015 23:25:00 -0800 + +bitcoin (0.12.0-trusty4) trusty; urgency=medium + + * Fix libevent-dev dep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 23 Feb 2015 23:25:00 -0800 + +bitcoin (0.12.0-trusty3) trusty; urgency=medium + + * Fix precise boost dep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 23 Feb 2015 19:55:00 -0800 + +bitcoin (0.12.0-trusty2) trusty; urgency=medium + + * Fix libevent dep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 23 Feb 2015 19:53:00 -0800 + +bitcoin (0.12.0-trusty1) trusty; urgency=medium + + * New upstream release + * Various updates to contrib/debian were merged, a few were not + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Tue, 23 Feb 2015 19:29:00 -0800 + +bitcoin (0.11.2-trusty1) trusty; urgency=low + + * New upstream release. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Fri, 13 Nov 2015 18:39:00 -0800 + +bitcoin (0.11.1-trusty2) trusty; urgency=low + + * Remove minupnpc builddep. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Wed, 14 Oct 2015 23:06:00 -1000 + +bitcoin (0.11.1-trusty1) trusty; urgency=high + + * New upstream release. + * Disable all UPnP support. + + -- Matt Corallo (BlueMatt) <matt@mattcorallo.com> Wed, 14 Oct 2015 13:57:00 -1000 + bitcoin (0.11.0-precise1) precise; urgency=medium * New upstream release. @@ -179,7 +298,7 @@ bitcoin (0.5.3-natty0) natty; urgency=low bitcoin (0.5.2-natty1) natty; urgency=low * Remove mentions on anonymity in package descriptions and manpage. - These should never have been there, bitcoin isn't anonymous without + These should never have been there, bitcoin isnt anonymous without a ton of work that virtually no users will ever be willing and capable of doing @@ -220,7 +339,7 @@ bitcoin (0.5.0~rc1-natty1) natty; urgency=low * Add test_bitcoin to build test * Fix clean - * Remove unnecessary build-dependancies + * Remove uneccessary build-dependancies -- Matt Corallo <matt@bluematt.me> Wed, 26 Oct 2011 14:37:18 -0400 @@ -380,7 +499,7 @@ bitcoin (0.3.20.01~dfsg-1) unstable; urgency=low bitcoin (0.3.19~dfsg-6) unstable; urgency=low - * Fix override aggressive optimizations. + * Fix override agressive optimizations. * Fix tighten build-dependencies to really fit backporting to Lenny: + Add fallback build-dependency on libdb4.6++-dev. + Tighten unversioned Boost build-dependencies to recent versions, diff --git a/contrib/debian/control b/contrib/debian/control index fce6bc0118..0d6ad25e24 100644 --- a/contrib/debian/control +++ b/contrib/debian/control @@ -1,27 +1,30 @@ Source: bitcoin Section: utils Priority: optional -Maintainer: Jonas Smedegaard <dr@jones.dk> -Uploaders: Micah Anderson <micah@debian.org> +Maintainer: Matt Corallo <matt@mattcorallo.com> +Uploaders: Matt Corallo <matt@mattcorallo.com> Build-Depends: debhelper, devscripts, automake, libtool, bash-completion, - libboost-system-dev (>> 1.35) | libboost-system1.35-dev, libdb4.8++-dev, libssl-dev, pkg-config, - libminiupnpc8-dev | libminiupnpc-dev (>> 1.6), - libboost-filesystem-dev (>> 1.35) | libboost-filesystem1.35-dev, - libboost-program-options-dev (>> 1.35) | libboost-program-options1.35-dev, - libboost-thread-dev (>> 1.35) | libboost-thread1.35-dev, - libboost-test-dev (>> 1.35) | libboost-test1.35-dev, - qt4-qmake, - libqt4-dev, + libevent-dev, + libboost-system1.48-dev | libboost-system-dev (>> 1.47), + libboost-filesystem1.48-dev | libboost-filesystem-dev (>> 1.47), + libboost-program-options1.48-dev | libboost-program-options-dev (>> 1.47), + libboost-thread1.48-dev | libboost-thread-dev (>> 1.47), + libboost-test1.48-dev | libboost-test-dev (>> 1.47), + libboost-chrono1.48-dev | libboost-chrono-dev (>> 1.47), + libminiupnpc8-dev | libminiupnpc-dev, + qt4-qmake, libqt4-dev, + qttools5-dev-tools, qttools5-dev, libqrencode-dev, libprotobuf-dev, protobuf-compiler, - python + python, + libzmq3-dev Standards-Version: 3.9.2 Homepage: https://bitcoincore.org/ Vcs-Git: git://github.com/bitcoin/bitcoin.git @@ -31,11 +34,11 @@ Package: bitcoind Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} Description: peer-to-peer network based digital currency - daemon - Bitcoin is an experimental new digital currency that enables instant - payments to anyone, anywhere in the world. Bitcoin uses peer-to-peer - technology to operate with no central authority: managing transactions - and issuing money are carried out collectively by the network. Bitcoin Core - is the name of the open source software which enables the use of this currency. + Bitcoin is a free open source peer-to-peer electronic cash system that + is completely decentralized, without the need for a central server or + trusted parties. Users hold the crypto keys to their own money and + transact directly with each other, with the help of a P2P network to + check for double-spending. . This package provides the daemon, bitcoind, and the CLI tool bitcoin-cli to interact with the daemon. @@ -44,11 +47,11 @@ Package: bitcoin-qt Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} Description: peer-to-peer network based digital currency - Qt GUI - Bitcoin is an experimental new digital currency that enables instant - payments to anyone, anywhere in the world. Bitcoin uses peer-to-peer - technology to operate with no central authority: managing transactions - and issuing money are carried out collectively by the network. Bitcoin Core - is the name of the open source software which enables the use of this currency. + Bitcoin is a free open source peer-to-peer electronic cash system that + is completely decentralized, without the need for a central server or + trusted parties. Users hold the crypto keys to their own money and + transact directly with each other, with the help of a P2P network to + check for double-spending. . This package provides Bitcoin-Qt, a GUI for Bitcoin based on Qt. @@ -56,11 +59,11 @@ Package: bitcoin-tx Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} Description: peer-to-peer digital currency - standalone transaction tool - Bitcoin is an experimental new digital currency that enables instant - payments to anyone, anywhere in the world. Bitcoin uses peer-to-peer - technology to operate with no central authority: managing transactions - and issuing money are carried out collectively by the network. Bitcoin Core - is the name of the open source software which enables the use of this currency. + Bitcoin is a free open source peer-to-peer electronic cash system that + is completely decentralized, without the need for a central server or + trusted parties. Users hold the crypto keys to their own money and + transact directly with each other, with the help of a P2P network to + check for double-spending. . This package provides bitcoin-tx, a command-line transaction creation tool which can be used without a bitcoin daemon. Some means of diff --git a/contrib/debian/rules b/contrib/debian/rules index 3896d2caa3..6885e38521 100755 --- a/contrib/debian/rules +++ b/contrib/debian/rules @@ -12,10 +12,12 @@ override_dh_auto_clean: if [ -f Makefile ]; then $(MAKE) distclean; fi rm -rf Makefile.in aclocal.m4 configure src/Makefile.in src/bitcoin-config.h.in src/build-aux src/qt/Makefile.in src/qt/test/Makefile.in src/test/Makefile.in +QT=$(shell dpkg-vendor --derives-from Ubuntu && echo qt4 || echo qt5) + # Yea, autogen should be run on the source archive, but I like doing git archive override_dh_auto_configure: ./autogen.sh - ./configure + ./configure --with-gui=$(QT) override_dh_auto_test: make check diff --git a/contrib/devtools/commit-script-check.sh b/contrib/devtools/commit-script-check.sh new file mode 100755 index 0000000000..add4bb4883 --- /dev/null +++ b/contrib/devtools/commit-script-check.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# This simple script checks for commits beginning with: scripted-diff: +# If found, looks for a script between the lines -BEGIN VERIFY SCRIPT- and +# -END VERIFY SCRIPT-. If no ending is found, it reads until the end of the +# commit message. + +# The resulting script should exactly transform the previous commit into the current +# one. Any remaining diff signals an error. + +if test "x$1" = "x"; then + echo "Usage: $0 <commit>..." + exit 1 +fi + +RET=0 +PREV_BRANCH=`git name-rev --name-only HEAD` +PREV_HEAD=`git rev-parse HEAD` +for i in `git rev-list --reverse $1`; do + git rev-list -n 1 --pretty="%s" $i | grep -q "^scripted-diff:" || continue + git checkout --quiet $i^ || exit + SCRIPT="`git rev-list --format=%b -n1 $i | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d'`" + if test "x$SCRIPT" = "x"; then + echo "Error: missing script for: $i" + echo "Failed" + RET=1 + else + echo "Running script for: $i" + echo "$SCRIPT" + eval "$SCRIPT" + git --no-pager diff --exit-code $i && echo "OK" || (echo "Failed"; false) || RET=1 + fi + git reset --quiet --hard HEAD +done +git checkout --quiet $PREV_BRANCH 2>/dev/null || git checkout --quiet $PREV_HEAD +exit $RET diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 3fee39143d..e9816f7d19 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 Bitcoin Core Developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -127,6 +127,9 @@ def tree_sha512sum(commit='HEAD'): raise IOError('Non-zero return value executing git cat-file') return overall.hexdigest() +def print_merge_details(pull, title, branch, base_branch, head_branch): + print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) + subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) def parse_arguments(): epilog = ''' @@ -171,7 +174,7 @@ def main(): info = retrieve_pr_info(repo,pull) if info is None: exit(1) - title = info['title'] + title = info['title'].strip() # precedence order for destination branch argument: # - command line argument # - githubmerge.branch setting @@ -256,8 +259,7 @@ def main(): printf("ERROR: Cannot update message.",file=stderr) exit(4) - print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) - subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) + print_merge_details(pull, title, branch, base_branch, head_branch) print() # Run test command if configured. @@ -276,12 +278,6 @@ def main(): print("Difference with github ignored.",file=stderr) else: exit(6) - reply = ask_prompt("Press 'd' to accept the diff.") - if reply.lower() == 'd': - print("Diff accepted.",file=stderr) - else: - print("ERROR: Diff rejected.",file=stderr) - exit(6) else: # Verify the result manually. print("Dropping you on a shell so you can try building/testing the merged source.",file=stderr) @@ -290,12 +286,6 @@ def main(): if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt os.putenv('debian_chroot',pull) subprocess.call([BASH,'-i']) - reply = ask_prompt("Type 'm' to accept the merge.") - if reply.lower() == 'm': - print("Merge accepted.",file=stderr) - else: - print("ERROR: Merge rejected.",file=stderr) - exit(7) second_sha512 = tree_sha512sum() if first_sha512 != second_sha512: @@ -303,16 +293,18 @@ def main(): exit(8) # Sign the merge commit. - reply = ask_prompt("Type 's' to sign off on the merge.") - if reply == 's': - try: - subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) - except subprocess.CalledProcessError as e: - print("Error signing, exiting.",file=stderr) + print_merge_details(pull, title, branch, base_branch, head_branch) + while True: + reply = ask_prompt("Type 's' to sign off on the above merge, or 'x' to reject and exit.").lower() + if reply == 's': + try: + subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) + break + except subprocess.CalledProcessError as e: + print("Error while signing, asking again.",file=stderr) + elif reply == 'x': + print("Not signing off on merge, exiting.",file=stderr) exit(1) - else: - print("Not signing off on merge, exiting.",file=stderr) - exit(1) # Put the result in branch. subprocess.check_call([GIT,'checkout','-q',branch]) @@ -326,9 +318,13 @@ def main(): subprocess.call([GIT,'branch','-q','-D',local_merge_branch],stderr=devnull) # Push the result. - reply = ask_prompt("Type 'push' to push the result to %s, branch %s." % (host_repo,branch)) - if reply.lower() == 'push': - subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) + while True: + reply = ask_prompt("Type 'push' to push the result to %s, branch %s, or 'x' to exit without pushing." % (host_repo,branch)).lower() + if reply == 'push': + subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) + break + elif reply == 'x': + exit(1) if __name__ == '__main__': main() diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 00af4bdc6f..3da8510cfb 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -132,6 +132,7 @@ script: | export PATH=${WRAP_DIR}:${PATH} # Create the release tarball using (arbitrarily) the first host + export GIT_DIR="$PWD/.git" ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 05cc65414f..206db7c19e 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -101,6 +101,7 @@ script: | export PATH=${WRAP_DIR}:${PATH} # Create the release tarball using (arbitrarily) the first host + export GIT_DIR="$PWD/.git" ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 3388977e0d..1d4d70494b 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -116,6 +116,7 @@ script: | export PATH=${WRAP_DIR}:${PATH} # Create the release tarball using (arbitrarily) the first host + export GIT_DIR="$PWD/.git" ./autogen.sh CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/ make dist diff --git a/contrib/init/bitcoind.openrcconf b/contrib/init/bitcoind.openrcconf index 0cbff6d30d..f70e25cb5f 100644 --- a/contrib/init/bitcoind.openrcconf +++ b/contrib/init/bitcoind.openrcconf @@ -23,7 +23,7 @@ #BITCOIND_NICE=0 # Additional options (avoid -conf and -datadir, use flags above) -BITCOIND_OPTS="-disablewallet" +#BITCOIND_OPTS="" # The timeout in seconds OpenRC will wait for bitcoind to terminate # after a SIGTERM has been raised. diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 5995f9f438..23a568ad13 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -302,7 +302,6 @@ def copyFramework(framework, path, verbose): if os.path.exists(fromContentsDir): toContentsDir = os.path.join(path, framework.destinationVersionContentsDirectory) shutil.copytree(fromContentsDir, toContentsDir, symlinks=True) - contentslinkfrom = os.path.join(path, framework.destinationContentsDirectory) if verbose >= 3: print("Copied Contents:", fromContentsDir) print(" to:", toContentsDir) @@ -675,9 +674,8 @@ else: if verbose >= 2: print("+ Installing qt.conf +") -f = open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") -f.write(qt_conf.encode()) -f.close() +with open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") as f: + f.write(qt_conf.encode()) # ------------------------------------------------ diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md index afe902fd7f..139c03181f 100644 --- a/contrib/seeds/README.md +++ b/contrib/seeds/README.md @@ -8,7 +8,7 @@ and remove old versions as necessary. The seeds compiled into the release are created from sipa's DNS seed data, like this: - curl -s http://bitcoin.sipa.be/seeds.txt > seeds_main.txt + curl -s http://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt python3 makeseeds.py < seeds_main.txt > nodes_main.txt python3 generate-seeds.py . > ../../src/chainparamsseeds.h diff --git a/depends/config.guess b/depends/config.guess index bbd48b60e8..69ed3e573b 100755 --- a/depends/config.guess +++ b/depends/config.guess @@ -2,7 +2,7 @@ # Attempt to guess a canonical system name. # Copyright 1992-2017 Free Software Foundation, Inc. -timestamp='2017-01-01' +timestamp='2017-03-05' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -837,10 +837,11 @@ EOF UNAME_PROCESSOR=`/usr/bin/uname -p` case ${UNAME_PROCESSOR} in amd64) - echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; - *) - echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; esac + echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` exit ;; i*:CYGWIN*:*) echo ${UNAME_MACHINE}-pc-cygwin @@ -1343,6 +1344,9 @@ EOF NSR-?:NONSTOP_KERNEL:*:*) echo nsr-tandem-nsk${UNAME_RELEASE} exit ;; + NSX-?:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk${UNAME_RELEASE} + exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux exit ;; diff --git a/depends/config.sub b/depends/config.sub index 7e792b4ae1..40ea5dfe11 100755 --- a/depends/config.sub +++ b/depends/config.sub @@ -2,7 +2,7 @@ # Configuration validation subroutine script. # Copyright 1992-2017 Free Software Foundation, Inc. -timestamp='2017-01-01' +timestamp='2017-04-02' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -263,7 +263,7 @@ case $basic_machine in | fido | fr30 | frv | ft32 \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ - | i370 | i860 | i960 | ia64 \ + | i370 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ | le32 | le64 \ @@ -315,6 +315,7 @@ case $basic_machine in | ubicom32 \ | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ | visium \ + | wasm32 \ | we32k \ | x86 | xc16x | xstormy16 | xtensa \ | z8k | z80) @@ -388,7 +389,7 @@ case $basic_machine in | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ | hexagon-* \ - | i*86-* | i860-* | i960-* | ia64-* \ + | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ | ip2k-* | iq2000-* \ | k1om-* \ | le32-* | le64-* \ @@ -446,6 +447,7 @@ case $basic_machine in | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ | vax-* \ | visium-* \ + | wasm32-* \ | we32k-* \ | x86-* | x86_64-* | xc16x-* | xps100-* \ | xstormy16-* | xtensa*-* \ @@ -948,6 +950,9 @@ case $basic_machine in nsr-tandem) basic_machine=nsr-tandem ;; + nsx-tandem) + basic_machine=nsx-tandem + ;; op50n-* | op60c-*) basic_machine=hppa1.1-oki os=-proelf @@ -1243,6 +1248,9 @@ case $basic_machine in basic_machine=a29k-wrs os=-vxworks ;; + wasm32) + basic_machine=wasm32-unknown + ;; w65*) basic_machine=w65-wdc os=-none diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 57d96e4821..bf773ccd14 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,8 +1,8 @@ package=boost -$(package)_version=1_63_0 -$(package)_download_path=https://sourceforge.net/projects/boost/files/boost/1.63.0 +$(package)_version=1_64_0 +$(package)_download_path=https://dl.bintray.com/boostorg/release/1.64.0/source/ $(package)_file_name=$(package)_$($(package)_version).tar.bz2 -$(package)_sha256_hash=beae2529f759f6b3bf3f4969a19c2e9d6f0c503edcb2de4a61d1428519fcb3b0 +$(package)_sha256_hash=7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 define $(package)_set_vars $(package)_config_opts_release=variant=release diff --git a/depends/packages/dbus.mk b/depends/packages/dbus.mk index 90ddcb923f..bbe0375409 100644 --- a/depends/packages/dbus.mk +++ b/depends/packages/dbus.mk @@ -1,8 +1,8 @@ package=dbus -$(package)_version=1.10.14 -$(package)_download_path=http://dbus.freedesktop.org/releases/dbus +$(package)_version=1.10.18 +$(package)_download_path=https://dbus.freedesktop.org/releases/dbus $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=23238f70353e38ce5ca183ebc9525c0d97ac00ef640ad29cf794782af6e6a083 +$(package)_sha256_hash=6049ddd5f3f3e2618f615f1faeda0a115104423a7996b7aa73e2f36e38cc514a $(package)_dependencies=expat define $(package)_set_vars diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 70f345e71d..00231d75d5 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -1,8 +1,8 @@ package=libevent -$(package)_version=2.1.7 +$(package)_version=2.1.8-stable $(package)_download_path=https://github.com/libevent/libevent/archive/ -$(package)_file_name=release-$($(package)_version)-rc.tar.gz -$(package)_sha256_hash=548362d202e22fe24d4c3fad38287b4f6d683e6c21503341373b89785fa6f991 +$(package)_file_name=release-$($(package)_version).tar.gz +$(package)_sha256_hash=316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d define $(package)_preprocess_cmds ./autogen.sh diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index e34cf7be2f..1bb8cb5d26 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -1,8 +1,8 @@ package=miniupnpc -$(package)_version=2.0 +$(package)_version=2.0.20170509 $(package)_download_path=http://miniupnp.free.fr/files $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=d434ceb8986efbe199c5ca53f90ed53eab290b1e6d0530b717eb6fa49d61f93b +$(package)_sha256_hash=d3c368627f5cdfb66d3ebd64ca39ba54d6ff14a61966dbecb8dd296b7039f16a define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" diff --git a/depends/packages/native_ccache.mk b/depends/packages/native_ccache.mk index 4ed61a49e9..966804ce8b 100644 --- a/depends/packages/native_ccache.mk +++ b/depends/packages/native_ccache.mk @@ -1,8 +1,8 @@ package=native_ccache -$(package)_version=3.3.3 +$(package)_version=3.3.4 $(package)_download_path=https://samba.org/ftp/ccache $(package)_file_name=ccache-$($(package)_version).tar.bz2 -$(package)_sha256_hash=2985bc5e32ebe38d2958d508eb54ddcad39eed909489c0c2988035214597ca54 +$(package)_sha256_hash=fa9d7f38367431bc86b19ad107d709ca7ecf1574fdacca01698bdf0a47cd8567 define $(package)_set_vars $(package)_config_opts= diff --git a/doc/build-osx.md b/doc/build-osx.md index a15bcd012c..32d7dbd69e 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -11,14 +11,14 @@ Install the OS X command line tools: When the popup appears, click `Install`. -Then install [Homebrew](http://brew.sh). +Then install [Homebrew](https://brew.sh). Dependencies ---------------------- - brew install automake berkeley-db4 libtool boost --c++11 miniupnpc openssl pkg-config protobuf --c++11 qt5 libevent + brew install automake berkeley-db4 libtool boost --c++11 miniupnpc openssl pkg-config protobuf qt libevent -In case you want to build the disk image with `make deploy` (.dmg / optional), you need RSVG +If you want to build the disk image with `make deploy` (.dmg / optional), you need RSVG brew install librsvg diff --git a/doc/build-unix.md b/doc/build-unix.md index 31a88a1b18..b7eae2a630 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -321,8 +321,10 @@ Clang is installed by default as `cc` compiler, this makes it easier to get started than on [OpenBSD](build-openbsd.md). Installing dependencies: pkg install autoconf automake libtool pkgconf - pkg install boost-libs openssl libevent2 + pkg install boost-libs openssl libevent + pkg install gmake +You need to use GNU make (`gmake`) instead of `make`. (`libressl` instead of `openssl` will also work) For the wallet (optional): @@ -338,7 +340,7 @@ Then build using: ./autogen.sh ./configure --with-incompatible-bdb BDB_CFLAGS="-I/usr/local/include/db5" BDB_LIBS="-L/usr/local/lib -ldb_cxx-5" - make + gmake *Note on debugging*: The version of `gdb` installed by default is [ancient and considered harmful](https://wiki.freebsd.org/GdbRetirement). It is not suitable for debugging a multi-threaded C++ program, not even for getting backtraces. Please install the package `gdb` and diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 797507cd3e..ec6abda91e 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -3,41 +3,64 @@ Developer Notes Various coding styles have been used during the history of the codebase, and the result is not very consistent. However, we're now trying to converge to -a single style, so please use it in new code. Old code will be converted -gradually and you are encouraged to use the provided -[clang-format-diff script](/contrib/devtools/README.md#clang-format-diffpy) -to clean up the patch automatically before submitting a pull request. +a single style, which is specified below. When writing patches, favor the new +style over attempting to mimick the surrounding style, except for move-only +commits. + +Do not submit patches solely to modify the style of existing code. -- Basic rules specified in [src/.clang-format](/src/.clang-format). +- **Indentation and whitespace rules** as specified in +[src/.clang-format](/src/.clang-format). You can use the provided +[clang-format-diff script](/contrib/devtools/README.md#clang-format-diffpy) +tool to clean up patches automatically before submission. - Braces on new lines for namespaces, classes, functions, methods. - Braces on the same line for everything else. - 4 space indentation (no tabs) for every block except namespaces. - No indentation for `public`/`protected`/`private` or for `namespace`. - No extra spaces inside parenthesis; don't do ( this ) - No space after function names; one space after `if`, `for` and `while`. - - If an `if` only has a single-statement then-clause, it can appear - on the same line as the if, without braces. In every other case, - braces are required, and the then and else clauses must appear + - If an `if` only has a single-statement `then`-clause, it can appear + on the same line as the `if`, without braces. In every other case, + braces are required, and the `then` and `else` clauses must appear correctly indented on a new line. + +- **Symbol naming conventions**. These are preferred in new code, but are not +required when doing so would need changes to significant pieces of existing +code. + - Variable and namespace names are all lowercase, and may use `_` to + separate words. + - Class member variables have a `m_` prefix. + - Global variables have a `g_` prefix. + - Constant names are all uppercase, and use `_` to separate words. + - Class names, function names and method names are CamelCase. Do not prefix + class names with `C`. + +- **Miscellaneous** - `++i` is preferred over `i++`. Block style example: ```c++ +int g_count = 0; + namespace foo { class Class { + std::string m_name; + +public: bool Function(const std::string& s, int n) { // Comment summarising what this section of code does for (int i = 0; i < n; ++i) { + int total_sum = 0; // When something fails, return early if (!Something()) return false; ... - if (SomethingElse()) { - DoMore(); + if (SomethingElse(i)) { + total_sum += ComputeSomething(g_count); } else { - DoLess(); + DoSomething(m_name, total_sum); } } @@ -343,10 +366,9 @@ Strings and formatting Variable names -------------- -The shadowing warning (`-Wshadow`) is enabled by default. It prevents issues rising -from using a different variable with the same name. - -Please name variables so that their names do not shadow variables defined in the source code. +Although the shadowing warning (`-Wshadow`) is not enabled by default (it prevents issues rising +from using a different variable with the same name), +please name variables so that their names do not shadow variables defined in the source code. E.g. in member initializers, prepend `_` to the argument name shadowing the member name: @@ -495,3 +517,76 @@ Git and GitHub tips This will add an `upstream-pull` remote to your git repository, which can be fetched using `git fetch --all` or `git fetch upstream-pull`. Afterwards, you can use `upstream-pull/NUMBER/head` in arguments to `git show`, `git checkout` and anywhere a commit id would be acceptable to see the changes from pull request NUMBER. + +RPC interface guidelines +-------------------------- + +A few guidelines for introducing and reviewing new RPC interfaces: + +- Method naming: use consecutive lower-case names such as `getrawtransaction` and `submitblock` + + - *Rationale*: Consistency with existing interface. + +- Argument naming: use snake case `fee_delta` (and not, e.g. camel case `feeDelta`) + + - *Rationale*: Consistency with existing interface. + +- Use the JSON parser for parsing, don't manually parse integers or strings from + arguments unless absolutely necessary. + + - *Rationale*: Introduces hand-rolled string manipulation code at both the caller and callee sites, + which is error prone, and it is easy to get things such as escaping wrong. + JSON already supports nested data structures, no need to re-invent the wheel. + + - *Exception*: AmountToValue can parse amounts as string. This was introduced because many JSON + parsers and formatters hard-code handling decimal numbers as floating point + values, resulting in potential loss of precision. This is unacceptable for + monetary values. **Always** use `AmountToValue` and `ValueToAmount` when + inputting or outputting monetary values. The only exceptions to this are + `prioritisetransaction` and `getblocktemplate` because their interface + is specified as-is in BIP22. + +- Missing arguments and 'null' should be treated the same: as default values. If there is no + default value, both cases should fail in the same way. + + - *Rationale*: Avoids surprises when switching to name-based arguments. Missing name-based arguments + are passed as 'null'. + + - *Exception*: Many legacy exceptions to this exist, one of the worst ones is + `getbalance` which follows a completely different code path based on the + number of arguments. We are still in the process of cleaning these up. Do not introduce + new ones. + +- Try not to overload methods on argument type. E.g. don't make `getblock(true)` and `getblock("hash")` + do different things. + + - *Rationale*: This is impossible to use with `bitcoin-cli`, and can be surprising to users. + + - *Exception*: Some RPC calls can take both an `int` and `bool`, most notably when a bool was switched + to a multi-value, or due to other historical reasons. **Always** have false map to 0 and + true to 1 in this case. + +- Don't forget to fill in the argument names correctly in the RPC command table. + + - *Rationale*: If not, the call can not be used with name-based arguments. + +- Set okSafeMode in the RPC command table to a sensible value: safe mode is when the + blockchain is regarded to be in a confused state, and the client deems it unsafe to + do anything irreversible such as send. Anything that just queries should be permitted. + + - *Rationale*: Troubleshooting a node in safe mode is difficult if half the + RPCs don't work. + +- Add every non-string RPC argument `(method, idx, name)` to the table `vRPCConvertParams` in `rpc/client.cpp`. + + - *Rationale*: `bitcoin-cli` and the GUI debug console use this table to determine how to + convert a plaintext command line to JSON. If the types don't match, the method can be unusable + from there. + +- A RPC method must either be a wallet method or a non-wallet method. Do not + introduce new methods such as `getinfo` and `signrawtransaction` that differ + in behavior based on presence of a wallet. + + - *Rationale*: as well as complicating the implementation and interfering + with the introduction of multi-wallet, wallet and non-wallet code should be + separated to avoid introducing circular dependencies between code units. diff --git a/doc/fuzzing.md b/doc/fuzzing.md index bf3ad17861..5dedcb51c8 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -32,6 +32,13 @@ We disable ccache because we don't want to pollute the ccache with instrumented objects, and similarly don't want to use non-instrumented cached objects linked in. +The fuzzing can be sped up significantly (~200x) by using `afl-clang-fast` and +`afl-clang-fast++` in place of `afl-gcc` and `afl-g++` when compiling. When +compiling using `afl-clang-fast`/`afl-clang-fast++` the resulting +`test_bitcoin_fuzzy` binary will be instrumented in such a way that the AFL +features "persistent mode" and "deferred forkserver" can be used. See +https://github.com/mcarpenter/afl/tree/master/llvm_mode for details. + Preparing fuzzing ------------------ @@ -63,4 +70,3 @@ $AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/test_bitcoin_fuzzy You may have to change a few kernel parameters to test optimally - `afl-fuzz` will print an error and suggestion if so. - diff --git a/doc/release-notes.md b/doc/release-notes.md index af792118d6..075c7c3911 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -36,6 +36,15 @@ Notable changes Low-level RPC changes --------------------- +- The new database model no longer stores information about transaction + versions of unspent outputs. This means that: + - The `gettxout` RPC no longer has a `version` field in the response. + - The `gettxoutsetinfo` RPC reports `hash_serialized_2` instead of `hash_serialized`, + which does not commit to the transaction versions of unspent outputs, but does + commit to the height and coinbase information. + - The `getutxos` REST path no longer reports the `txvers` field in JSON format, + and always reports 0 for transaction versions in the binary format + - Error codes have been updated to be more accurate for the following error cases: - `getblock` now returns RPC_MISC_ERROR if the block can't be found on disk (for example if the block has been pruned). Previously returned RPC_INTERNAL_ERROR. diff --git a/doc/release-notes/release-notes-0.14.1.md b/doc/release-notes/release-notes-0.14.1.md new file mode 100644 index 0000000000..ef072afd4d --- /dev/null +++ b/doc/release-notes/release-notes-0.14.1.md @@ -0,0 +1,143 @@ +Bitcoin Core version 0.14.1 is now available from: + + <https://bitcoin.org/bin/bitcoin-core-0.14.1/> + +This is a new minor version release, including various bugfixes and +performance improvements, as well as updated translations. + +Please report bugs using the issue tracker at github: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +Compatibility +============== + +Bitcoin Core is extensively tested on multiple operating systems using +the Linux kernel, macOS 10.8+, and Windows Vista and later. + +Microsoft ended support for Windows XP on [April 8th, 2014](https://www.microsoft.com/en-us/WindowsForBusiness/end-of-xp-support), +No attempt is made to prevent installing or running the software on Windows XP, you +can still do so at your own risk but be aware that there are known instabilities and issues. +Please do not report issues about Windows XP to the issue tracker. + +Bitcoin Core should also work on most other Unix-like systems but is not +frequently tested on them. + +Notable changes +=============== + +RPC changes +----------- + +- The first positional argument of `createrawtransaction` was renamed from + `transactions` to `inputs`. + +- The argument of `disconnectnode` was renamed from `node` to `address`. + +These interface changes break compatibility with 0.14.0, when the named +arguments functionality, introduced in 0.14.0, is used. Client software +using these calls with named arguments needs to be updated. + +Mining +------ + +In previous versions, getblocktemplate required segwit support from downstream +clients/miners once the feature activated on the network. In this version, it +now supports non-segwit clients even after activation, by removing all segwit +transactions from the returned block template. This allows non-segwit miners to +continue functioning correctly even after segwit has activated. + +Due to the limitations in previous versions, getblocktemplate also recommended +non-segwit clients to not signal for the segwit version-bit. Since this is no +longer an issue, getblocktemplate now always recommends signalling segwit for +all miners. This is safe because ability to enforce the rule is the only +required criteria for safe activation, not actually producing segwit-enabled +blocks. + +UTXO memory accounting +---------------------- + +Memory usage for the UTXO cache is being calculated more accurately, so that +the configured limit (`-dbcache`) will be respected when memory usage peaks +during cache flushes. The memory accounting in prior releases is estimated to +only account for half the actual peak utilization. + +The default `-dbcache` has also been changed in this release to 450MiB. Users +who currently set `-dbcache` to a high value (e.g. to keep the UTXO more fully +cached in memory) should consider increasing this setting in order to achieve +the same cache performance as prior releases. Users on low-memory systems +(such as systems with 1GB or less) should consider specifying a lower value for +this parameter. + +Additional information relating to running on low-memory systems can be found +here: +[reducing-bitcoind-memory-usage.md](https://gist.github.com/laanwj/efe29c7661ce9b6620a7). + +0.14.1 Change log +================= + +Detailed release notes follow. This overview includes changes that affect +behavior, not code moves, refactors and string updates. For convenience in locating +the code changes and accompanying discussion, both the pull request and +git merge commit are mentioned. + +### RPC and other APIs +- #10084 `142fbb2` Rename first named arg of createrawtransaction (MarcoFalke) +- #10139 `f15268d` Remove auth cookie on shutdown (practicalswift) +- #10146 `2fea10a` Better error handling for submitblock (rawodb, gmaxwell) +- #10144 `d947afc` Prioritisetransaction wasn't always updating ancestor fee (sdaftuar) +- #10204 `3c79602` Rename disconnectnode argument (jnewbery) + +### Block and transaction handling +- #10126 `0b5e162` Compensate for memory peak at flush time (sipa) +- #9912 `fc3d7db` Optimize GetWitnessHash() for non-segwit transactions (sdaftuar) +- #10133 `ab864d3` Clean up calculations of pcoinsTip memory usage (morcos) + +### P2P protocol and network code +- #9953/#10013 `d2548a4` Fix shutdown hang with >= 8 -addnodes set (TheBlueMatt) +- #10176 `30fa231` net: gracefully handle NodeId wrapping (theuni) + +### Build system +- #9973 `e9611d1` depends: fix zlib build on osx (theuni) + +### GUI +- #10060 `ddc2dd1` Ensure an item exists on the rpcconsole stack before adding (achow101) + +### Mining +- #9955/#10006 `569596c` Don't require segwit in getblocktemplate for segwit signalling or mining (sdaftuar) +- #9959/#10127 `b5c3440` Prevent slowdown in CreateNewBlock on large mempools (sdaftuar) + +### Tests and QA +- #10157 `55f641c` Fix the `mempool_packages.py` test (sdaftuar) + +### Miscellaneous +- #10037 `4d8e660` Trivial: Fix typo in help getrawtransaction RPC (keystrike) +- #10120 `e4c9a90` util: Work around (virtual) memory exhaustion on 32-bit w/ glibc (laanwj) +- #10130 `ecc5232` bitcoin-tx input verification (awemany, jnewbery) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Alex Morcos +- Andrew Chow +- Awemany +- Cory Fields +- Gregory Maxwell +- James Evans +- John Newbery +- MarcoFalke +- Matt Corallo +- Pieter Wuille +- practicalswift +- rawodb +- Suhas Daftuar +- Wladimir J. van der Laan + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). + diff --git a/share/genbuild.sh b/share/genbuild.sh index eecac4bd00..32ef2a5755 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -17,9 +17,13 @@ else exit 1 fi +git_check_in_repo() { + ! { git status --porcelain -uall --ignored "$@" 2>/dev/null || echo '??'; } | grep -q '?' +} + DESC="" SUFFIX="" -if [ -e "$(which git 2>/dev/null)" -a "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ]; then +if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" -a -e "$(which git 2>/dev/null)" -a "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ] && git_check_in_repo share/genbuild.sh; then # clean 'dirty' status of touched files that haven't been modified git diff >/dev/null 2>/dev/null diff --git a/src/Makefile.am b/src/Makefile.am index 69b5bb48ad..ae2eb29c94 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -95,6 +95,7 @@ BITCOIN_CORE_H = \ compat/sanity.h \ compressor.h \ consensus/consensus.h \ + consensus/tx_verify.h \ core_io.h \ core_memusage.h \ cuckoocache.h \ @@ -116,6 +117,7 @@ BITCOIN_CORE_H = \ netbase.h \ netmessagemaker.h \ noui.h \ + policy/feerate.h \ policy/fees.h \ policy/policy.h \ policy/rbf.h \ @@ -178,12 +180,13 @@ libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h libbitcoin_server_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_server_a_SOURCES = \ - addrman.cpp \ addrdb.cpp \ + addrman.cpp \ bloom.cpp \ blockencodings.cpp \ chain.cpp \ checkpoints.cpp \ + consensus/tx_verify.cpp \ httprpc.cpp \ httpserver.cpp \ init.cpp \ @@ -246,6 +249,8 @@ crypto_libbitcoin_crypto_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_a_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ + crypto/chacha20.h \ + crypto/chacha20.cpp \ crypto/common.h \ crypto/hmac_sha256.cpp \ crypto/hmac_sha256.h \ @@ -299,7 +304,6 @@ libbitcoin_consensus_a_SOURCES = \ libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ - amount.cpp \ base58.cpp \ chainparams.cpp \ coins.cpp \ @@ -310,6 +314,7 @@ libbitcoin_common_a_SOURCES = \ keystore.cpp \ netaddress.cpp \ netbase.cpp \ + policy/feerate.cpp \ protocol.cpp \ scheduler.cpp \ script/sign.cpp \ @@ -462,6 +467,14 @@ DISTCLEANFILES = obj/build.h EXTRA_DIST = $(CTAES_DIST) + +config/bitcoin-config.h: config/stamp-h1 + @$(MAKE) -C $(top_builddir) $(subdir)/$(@) +config/stamp-h1: $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in $(top_builddir)/config.status + $(AM_V_at)$(MAKE) -C $(top_builddir) $(subdir)/$(@) +$(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) + $(AM_V_at)$(MAKE) -C $(top_srcdir) $(subdir)/config/bitcoin-config.h.in + clean-local: -$(MAKE) -C secp256k1 clean -$(MAKE) -C univalue clean diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 948e13a9e1..391b9ebdf6 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -46,7 +46,8 @@ qt_test_test_bitcoin_qt_SOURCES = \ if ENABLE_WALLET qt_test_test_bitcoin_qt_SOURCES += \ qt/test/paymentservertests.cpp \ - qt/test/wallettests.cpp + qt/test/wallettests.cpp \ + wallet/test/wallet_test_fixture.cpp endif nodist_qt_test_test_bitcoin_qt_SOURCES = $(TEST_QT_MOC_CPP) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d08c8bde5b..ee1c11ff1f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -57,8 +57,8 @@ BITCOIN_TESTS =\ test/policyestimator_tests.cpp \ test/pow_tests.cpp \ test/prevector_tests.cpp \ - test/random_tests.cpp \ test/raii_event_tests.cpp \ + test/random_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ @@ -78,6 +78,7 @@ BITCOIN_TESTS =\ test/testutil.cpp \ test/testutil.h \ test/timedata_tests.cpp \ + test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ test/txvalidationcache_tests.cpp \ test/versionbits_tests.cpp \ diff --git a/src/addrman.cpp b/src/addrman.cpp index ed9c128eb2..4a408b9beb 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -351,8 +351,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nKBucket = RandomInt(ADDRMAN_TRIED_BUCKET_COUNT); int nKBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); while (vvTried[nKBucket][nKBucketPos] == -1) { - nKBucket = (nKBucket + insecure_rand.rand32()) % ADDRMAN_TRIED_BUCKET_COUNT; - nKBucketPos = (nKBucketPos + insecure_rand.rand32()) % ADDRMAN_BUCKET_SIZE; + nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; + nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; } int nId = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nId) == 1); @@ -368,8 +368,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nUBucket = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); int nUBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); while (vvNew[nUBucket][nUBucketPos] == -1) { - nUBucket = (nUBucket + insecure_rand.rand32()) % ADDRMAN_NEW_BUCKET_COUNT; - nUBucketPos = (nUBucketPos + insecure_rand.rand32()) % ADDRMAN_BUCKET_SIZE; + nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; + nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; } int nId = vvNew[nUBucket][nUBucketPos]; assert(mapInfo.count(nId) == 1); diff --git a/src/addrman.h b/src/addrman.h index f123b20b1b..70d907488f 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -136,13 +136,13 @@ public: */ //! total number of buckets for tried addresses -#define ADDRMAN_TRIED_BUCKET_COUNT 256 +#define ADDRMAN_TRIED_BUCKET_COUNT_LOG2 8 //! total number of buckets for new addresses -#define ADDRMAN_NEW_BUCKET_COUNT 1024 +#define ADDRMAN_NEW_BUCKET_COUNT_LOG2 10 //! maximum allowed number of entries in buckets for new and tried addresses -#define ADDRMAN_BUCKET_SIZE 64 +#define ADDRMAN_BUCKET_SIZE_LOG2 6 //! over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread #define ADDRMAN_TRIED_BUCKETS_PER_GROUP 8 @@ -171,6 +171,11 @@ public: //! the maximum number of nodes to return in a getaddr call #define ADDRMAN_GETADDR_MAX 2500 +//! Convenience +#define ADDRMAN_TRIED_BUCKET_COUNT (1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2) +#define ADDRMAN_NEW_BUCKET_COUNT (1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2) +#define ADDRMAN_BUCKET_SIZE (1 << ADDRMAN_BUCKET_SIZE_LOG2) + /** * Stochastical (IP) address manager */ diff --git a/src/amount.h b/src/amount.h index 93060f7193..2bd367cba2 100644 --- a/src/amount.h +++ b/src/amount.h @@ -6,10 +6,7 @@ #ifndef BITCOIN_AMOUNT_H #define BITCOIN_AMOUNT_H -#include "serialize.h" - -#include <stdlib.h> -#include <string> +#include <stdint.h> /** Amount in satoshis (Can be negative) */ typedef int64_t CAmount; @@ -17,8 +14,6 @@ typedef int64_t CAmount; static const CAmount COIN = 100000000; static const CAmount CENT = 1000000; -extern const std::string CURRENCY_UNIT; - /** No amount larger than this (in satoshi) is valid. * * Note that this constant is *not* the total money supply, which in Bitcoin @@ -31,42 +26,4 @@ extern const std::string CURRENCY_UNIT; static const CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -/** - * Fee rate in satoshis per kilobyte: CAmount / kB - */ -class CFeeRate -{ -private: - CAmount nSatoshisPerK; // unit is satoshis-per-1,000-bytes -public: - /** Fee rate of 0 satoshis per kB */ - CFeeRate() : nSatoshisPerK(0) { } - explicit CFeeRate(const CAmount& _nSatoshisPerK): nSatoshisPerK(_nSatoshisPerK) { } - /** Constructor for a fee rate in satoshis per kB. The size in bytes must not exceed (2^63 - 1)*/ - CFeeRate(const CAmount& nFeePaid, size_t nBytes); - CFeeRate(const CFeeRate& other) { nSatoshisPerK = other.nSatoshisPerK; } - /** - * Return the fee in satoshis for the given size in bytes. - */ - CAmount GetFee(size_t nBytes) const; - /** - * Return the fee in satoshis for a size of 1000 bytes - */ - CAmount GetFeePerK() const { return GetFee(1000); } - friend bool operator<(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK < b.nSatoshisPerK; } - friend bool operator>(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK > b.nSatoshisPerK; } - friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } - friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } - friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } - CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } - std::string ToString() const; - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nSatoshisPerK); - } -}; - #endif // BITCOIN_AMOUNT_H diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index b0df3d2b04..33631d2d15 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -5,6 +5,7 @@ #include "bench.h" #include "perf.h" +#include <assert.h> #include <iostream> #include <iomanip> #include <sys/time.h> diff --git a/src/bench/bench.h b/src/bench/bench.h index f12a41126c..1f36f2a4bc 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,10 +5,11 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <functional> +#include <limits> #include <map> #include <string> -#include <boost/function.hpp> #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> @@ -59,7 +60,7 @@ namespace benchmark { bool KeepRunning(); }; - typedef boost::function<void(State&)> BenchFunction; + typedef std::function<void(State&)> BenchFunction; class BenchRunner { diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 1e8e3d462f..5aab3381fd 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -35,14 +35,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50 * CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21 * CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22 * CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index c6c932454a..195388839e 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -40,7 +40,7 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) char a = '\0'; stream.write(&a, 1); // Prevent compaction - Consensus::Params params = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); while (state.KeepRunning()) { CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here @@ -48,7 +48,7 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) assert(stream.Rewind(sizeof(block_bench::block413567))); CValidationState validationState; - assert(CheckBlock(block, validationState, params)); + assert(CheckBlock(block, validationState, chainParams->GetConsensus())); } } diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 6fa9fe4fe8..88a2a570f9 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -68,7 +68,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) PrevectorJob(){ } PrevectorJob(FastRandomContext& insecure_rand){ - p.resize(insecure_rand.rand32() % (PREVECTOR_SIZE*2)); + p.resize(insecure_rand.randrange(PREVECTOR_SIZE*2)); } bool operator()() { diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 06882f1514..42891f345b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -48,7 +48,7 @@ static void CoinSelection(benchmark::State& state) addCoin(1000 * COIN, wallet, vCoins); addCoin(3 * COIN, wallet, vCoins); - std::set<std::pair<const CWalletTx*, unsigned int> > setCoinsRet; + std::set<CInputCoin> setCoinsRet; CAmount nValueRet; bool success = wallet.SelectCoinsMinConf(1003 * COIN, 1, 6, 0, vCoins, setCoinsRet, nValueRet); assert(success); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 737d3572ae..2914a36c7b 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -7,6 +7,7 @@ #include "bench.h" #include "bloom.h" #include "hash.h" +#include "random.h" #include "uint256.h" #include "utiltime.h" #include "crypto/ripemd160.h" @@ -69,6 +70,28 @@ static void SipHash_32b(benchmark::State& state) } } +static void FastRandom_32bit(benchmark::State& state) +{ + FastRandomContext rng(true); + uint32_t x = 0; + while (state.KeepRunning()) { + for (int i = 0; i < 1000000; i++) { + x += rng.rand32(); + } + } +} + +static void FastRandom_1bit(benchmark::State& state) +{ + FastRandomContext rng(true); + uint32_t x = 0; + while (state.KeepRunning()) { + for (int i = 0; i < 1000000; i++) { + x += rng.randbool(); + } + } +} + BENCHMARK(RIPEMD160); BENCHMARK(SHA1); BENCHMARK(SHA256); @@ -76,3 +99,5 @@ BENCHMARK(SHA512); BENCHMARK(SHA256_32b); BENCHMARK(SipHash_32b); +BENCHMARK(FastRandom_32bit); +BENCHMARK(FastRandom_1bit); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 5edd43d41e..885b787b4d 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -30,6 +30,8 @@ static const int CONTINUE_EXECUTION=-1; std::string HelpMessageCli() { + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); std::string strUsage; strUsage += HelpMessageGroup(_("Options:")); strUsage += HelpMessageOpt("-?", _("This help message")); @@ -38,7 +40,7 @@ std::string HelpMessageCli() AppendParamsHelpMessages(strUsage); strUsage += HelpMessageOpt("-named", strprintf(_("Pass named instead of positional arguments (default: %s)"), DEFAULT_NAMED)); strUsage += HelpMessageOpt("-rpcconnect=<ip>", strprintf(_("Send commands to node running on <ip> (default: %s)"), DEFAULT_RPCCONNECT)); - strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Connect to JSON-RPC on <port> (default: %u or testnet: %u)"), BaseParams(CBaseChainParams::MAIN).RPCPort(), BaseParams(CBaseChainParams::TESTNET).RPCPort())); + strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Connect to JSON-RPC on <port> (default: %u or testnet: %u)"), defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort())); strUsage += HelpMessageOpt("-rpcwait", _("Wait for RPC server to start")); strUsage += HelpMessageOpt("-rpcuser=<user>", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections")); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 83b855cbcf..cf280f485c 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -556,24 +556,26 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (nOut < 0) throw std::runtime_error("vout must be positive"); + COutPoint out(txid, nOut); std::vector<unsigned char> pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { + const Coin& coin = view.AccessCoin(out); + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw std::runtime_error(err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(prevOut["amount"]); + newcoin.out.nValue = AmountFromValue(prevOut["amount"]); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and private keys given, @@ -595,13 +597,13 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (!coins || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsSpent()) { fComplete = false; continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: @@ -657,11 +659,13 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, MutateTxDelOutput(tx, commandVal); else if (command == "outaddr") MutateTxAddOutAddr(tx, commandVal); - else if (command == "outpubkey") + else if (command == "outpubkey") { + if (!ecc) { ecc.reset(new Secp256k1Init()); } MutateTxAddOutPubKey(tx, commandVal); - else if (command == "outmultisig") + } else if (command == "outmultisig") { + if (!ecc) { ecc.reset(new Secp256k1Init()); } MutateTxAddOutMultiSig(tx, commandVal); - else if (command == "outscript") + } else if (command == "outscript") MutateTxAddOutScript(tx, commandVal); else if (command == "outdata") MutateTxAddOutData(tx, commandVal); diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 31680a8ec7..5922e45801 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -117,17 +117,14 @@ bool AppInit(int argc, char* argv[]) return false; } - // Command-line RPC - bool fCommandLine = false; - for (int i = 1; i < argc; i++) - if (!IsSwitchChar(argv[i][0]) && !boost::algorithm::istarts_with(argv[i], "bitcoin:")) - fCommandLine = true; - - if (fCommandLine) - { - fprintf(stderr, "Error: There is no RPC client functionality in bitcoind anymore. Use the bitcoin-cli utility instead.\n"); - exit(EXIT_FAILURE); + // Error out when loose non-argument tokens are encountered on command line + for (int i = 1; i < argc; i++) { + if (!IsSwitchChar(argv[i][0])) { + fprintf(stderr, "Error: Command line contains unexpected token '%s', see bitcoind -h for a list of options.\n", argv[i]); + exit(EXIT_FAILURE); + } } + // -server defaults to true for bitcoind but not for the GUI so do this here SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console diff --git a/src/bloom.cpp b/src/bloom.cpp index ac3e565721..7ed982c984 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -19,7 +19,7 @@ #define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 #define LN2 0.6931471805599453094172321214581765680755001343602552 -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn, unsigned char nFlagsIn) : +CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn, unsigned char nFlagsIn) : /** * The ideal size for a bloom filter with a given number of elements and false positive rate is: * - nElements * log(fp rate) / ln(2)^2 @@ -40,7 +40,7 @@ CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int } // Private constructor used by CRollingBloomFilter -CBloomFilter::CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweakIn) : +CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn) : vData((unsigned int)(-1 / LN2SQUARED * nElements * log(nFPRate)) / 8), isFull(false), isEmpty(true), @@ -120,7 +120,7 @@ void CBloomFilter::clear() isEmpty = true; } -void CBloomFilter::reset(unsigned int nNewTweak) +void CBloomFilter::reset(const unsigned int nNewTweak) { clear(); nTweak = nNewTweak; @@ -214,7 +214,7 @@ void CBloomFilter::UpdateEmptyFull() isEmpty = empty; } -CRollingBloomFilter::CRollingBloomFilter(unsigned int nElements, double fpRate) +CRollingBloomFilter::CRollingBloomFilter(const unsigned int nElements, const double fpRate) { double logFpRate = log(fpRate); /* The optimal number of hash functions is log(fpRate) / log(0.5), but diff --git a/src/bloom.h b/src/bloom.h index 5ad727c330..7ca9682239 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -54,7 +54,7 @@ private: unsigned int Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const; // Private constructor for CRollingBloomFilter, no restrictions on size - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak); + CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak); friend class CRollingBloomFilter; public: @@ -67,7 +67,7 @@ public: * It should generally always be a random value (and is largely only exposed for unit testing) * nFlags should be one of the BLOOM_UPDATE_* enums (not _MASK) */ - CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak, unsigned char nFlagsIn); + CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak, unsigned char nFlagsIn); CBloomFilter() : isFull(true), isEmpty(false), nHashFuncs(0), nTweak(0), nFlags(0) {} ADD_SERIALIZE_METHODS; @@ -89,7 +89,7 @@ public: bool contains(const uint256& hash) const; void clear(); - void reset(unsigned int nNewTweak); + void reset(const unsigned int nNewTweak); //! True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS //! (catch a filter which was just deserialized which was too big) @@ -122,7 +122,7 @@ public: // A random bloom filter calls GetRand() at creation time. // Don't create global CRollingBloomFilter objects, as they may be // constructed before the randomizer is properly initialized. - CRollingBloomFilter(unsigned int nElements, double nFPRate); + CRollingBloomFilter(const unsigned int nElements, const double nFPRate); void insert(const std::vector<unsigned char>& vKey); void insert(const uint256& hash); diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 1dc29826af..5055fb3e0a 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -55,6 +55,12 @@ static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits return CreateGenesisBlock(pszTimestamp, genesisOutputScript, nTime, nNonce, nBits, nVersion, genesisReward); } +void CChainParams::UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +{ + consensus.vDeployments[d].nStartTime = nStartTime; + consensus.vDeployments[d].nTimeout = nTimeout; +} + /** * Main network */ @@ -165,7 +171,6 @@ public: }; } }; -static CMainParams mainParams; /** * Testnet (v3) @@ -253,7 +258,6 @@ public: } }; -static CTestNetParams testNetParams; /** * Regression test @@ -326,42 +330,34 @@ public: base58Prefixes[EXT_PUBLIC_KEY] = boost::assign::list_of(0x04)(0x35)(0x87)(0xCF).convert_to_container<std::vector<unsigned char> >(); base58Prefixes[EXT_SECRET_KEY] = boost::assign::list_of(0x04)(0x35)(0x83)(0x94).convert_to_container<std::vector<unsigned char> >(); } - - void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) - { - consensus.vDeployments[d].nStartTime = nStartTime; - consensus.vDeployments[d].nTimeout = nTimeout; - } }; -static CRegTestParams regTestParams; -static CChainParams *pCurrentParams = 0; +static std::unique_ptr<CChainParams> globalChainParams; const CChainParams &Params() { - assert(pCurrentParams); - return *pCurrentParams; + assert(globalChainParams); + return *globalChainParams; } -CChainParams& Params(const std::string& chain) +std::unique_ptr<CChainParams> CreateChainParams(const std::string& chain) { if (chain == CBaseChainParams::MAIN) - return mainParams; + return std::unique_ptr<CChainParams>(new CMainParams()); else if (chain == CBaseChainParams::TESTNET) - return testNetParams; + return std::unique_ptr<CChainParams>(new CTestNetParams()); else if (chain == CBaseChainParams::REGTEST) - return regTestParams; - else - throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); + return std::unique_ptr<CChainParams>(new CRegTestParams()); + throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); } void SelectParams(const std::string& network) { SelectBaseParams(network); - pCurrentParams = &Params(network); + globalChainParams = CreateChainParams(network); } -void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) { - regTestParams.UpdateBIP9Parameters(d, nStartTime, nTimeout); + globalChainParams->UpdateBIP9Parameters(d, nStartTime, nTimeout); } diff --git a/src/chainparams.h b/src/chainparams.h index 4fe88c691c..e5312d1080 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -11,6 +11,7 @@ #include "primitives/block.h" #include "protocol.h" +#include <memory> #include <vector> struct CDNSSeedData { @@ -75,6 +76,7 @@ public: const std::vector<SeedSpec6>& FixedSeeds() const { return vFixedSeeds; } const CCheckpointData& Checkpoints() const { return checkpointData; } const ChainTxData& TxData() const { return chainTxData; } + void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); protected: CChainParams() {} @@ -95,15 +97,17 @@ protected: }; /** - * Return the currently selected parameters. This won't change after app - * startup, except for unit tests. + * Creates and returns a std::unique_ptr<CChainParams> of the chosen chain. + * @returns a CChainParams* of the chosen chain. + * @throws a std::runtime_error if the chain is not supported. */ -const CChainParams &Params(); +std::unique_ptr<CChainParams> CreateChainParams(const std::string& chain); /** - * @returns CChainParams for the given BIP70 chain name. + * Return the currently selected parameters. This won't change after app + * startup, except for unit tests. */ -CChainParams& Params(const std::string& chain); +const CChainParams &Params(); /** * Sets the params returned by Params() to those for the given BIP70 chain name. @@ -114,6 +118,6 @@ void SelectParams(const std::string& chain); /** * Allows modifying the BIP9 regtest parameters. */ -void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); +void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index cb71a8b550..43c9a13c54 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -35,7 +35,6 @@ public: nRPCPort = 8332; } }; -static CBaseMainParams mainParams; /** * Testnet (v3) @@ -49,7 +48,6 @@ public: strDataDir = "testnet3"; } }; -static CBaseTestNetParams testNetParams; /* * Regression test @@ -63,31 +61,30 @@ public: strDataDir = "regtest"; } }; -static CBaseRegTestParams regTestParams; -static CBaseChainParams* pCurrentBaseParams = 0; +static std::unique_ptr<CBaseChainParams> globalChainBaseParams; const CBaseChainParams& BaseParams() { - assert(pCurrentBaseParams); - return *pCurrentBaseParams; + assert(globalChainBaseParams); + return *globalChainBaseParams; } -CBaseChainParams& BaseParams(const std::string& chain) +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain) { if (chain == CBaseChainParams::MAIN) - return mainParams; + return std::unique_ptr<CBaseChainParams>(new CBaseMainParams()); else if (chain == CBaseChainParams::TESTNET) - return testNetParams; + return std::unique_ptr<CBaseChainParams>(new CBaseTestNetParams()); else if (chain == CBaseChainParams::REGTEST) - return regTestParams; + return std::unique_ptr<CBaseChainParams>(new CBaseRegTestParams()); else throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); } void SelectBaseParams(const std::string& chain) { - pCurrentBaseParams = &BaseParams(chain); + globalChainBaseParams = CreateBaseChainParams(chain); } std::string ChainNameFromCommandLine() @@ -103,8 +100,3 @@ std::string ChainNameFromCommandLine() return CBaseChainParams::TESTNET; return CBaseChainParams::MAIN; } - -bool AreBaseParamsConfigured() -{ - return pCurrentBaseParams != NULL; -} diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index 59493afb9b..fc101f5b77 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_CHAINPARAMSBASE_H #define BITCOIN_CHAINPARAMSBASE_H +#include <memory> #include <string> #include <vector> @@ -31,6 +32,13 @@ protected: }; /** + * Creates and returns a std::unique_ptr<CBaseChainParams> of the chosen chain. + * @returns a CBaseChainParams* of the chosen chain. + * @throws a std::runtime_error if the chain is not supported. + */ +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain); + +/** * Append the help messages for the chainparams options to the * parameter string. */ @@ -42,8 +50,6 @@ void AppendParamsHelpMessages(std::string& strUsage, bool debugHelp=true); */ const CBaseChainParams& BaseParams(); -CBaseChainParams& BaseParams(const std::string& chain); - /** Sets the params returned by Params() to those for the given network. */ void SelectBaseParams(const std::string& chain); @@ -53,10 +59,4 @@ void SelectBaseParams(const std::string& chain); */ std::string ChainNameFromCommandLine(); -/** - * Return true if SelectBaseParamsFromCommandLine() has been called to select - * a network. - */ -bool AreBaseParamsConfigured(); - #endif // BITCOIN_CHAINPARAMSBASE_H diff --git a/src/checkqueue.h b/src/checkqueue.h index ea12df66dd..63c104c02a 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CHECKQUEUE_H #define BITCOIN_CHECKQUEUE_H +#include "sync.h" + #include <algorithm> #include <vector> diff --git a/src/coins.cpp b/src/coins.cpp index b2e33abf33..5b7c562678 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -4,174 +4,126 @@ #include "coins.h" +#include "consensus/consensus.h" #include "memusage.h" #include "random.h" #include <assert.h> -/** - * calculate number of bytes for the bitmask, and its number of non-zero bytes - * each bit in the bitmask represents the availability of one output, but the - * availabilities of the first two outputs are encoded separately - */ -void CCoins::CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const { - unsigned int nLastUsedByte = 0; - for (unsigned int b = 0; 2+b*8 < vout.size(); b++) { - bool fZero = true; - for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) { - if (!vout[2+b*8+i].IsNull()) { - fZero = false; - continue; - } - } - if (!fZero) { - nLastUsedByte = b + 1; - nNonzeroBytes++; - } - } - nBytes += nLastUsedByte; -} - -bool CCoins::Spend(uint32_t nPos) -{ - if (nPos >= vout.size() || vout[nPos].IsNull()) - return false; - vout[nPos].SetNull(); - Cleanup(); - return true; -} - -bool CCoinsView::GetCoins(const uint256 &txid, CCoins &coins) const { return false; } -bool CCoinsView::HaveCoins(const uint256 &txid) const { return false; } +bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } +bool CCoinsView::HaveCoin(const COutPoint &outpoint) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } bool CCoinsView::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return false; } CCoinsViewCursor *CCoinsView::Cursor() const { return 0; } CCoinsViewBacked::CCoinsViewBacked(CCoinsView *viewIn) : base(viewIn) { } -bool CCoinsViewBacked::GetCoins(const uint256 &txid, CCoins &coins) const { return base->GetCoins(txid, coins); } -bool CCoinsViewBacked::HaveCoins(const uint256 &txid) const { return base->HaveCoins(txid); } +bool CCoinsViewBacked::GetCoin(const COutPoint &outpoint, Coin &coin) const { return base->GetCoin(outpoint, coin); } +bool CCoinsViewBacked::HaveCoin(const COutPoint &outpoint) const { return base->HaveCoin(outpoint); } uint256 CCoinsViewBacked::GetBestBlock() const { return base->GetBestBlock(); } void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } +size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); } -SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} +SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} -CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } - -CCoinsViewCache::~CCoinsViewCache() -{ - assert(!hasModifier); -} +CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {} size_t CCoinsViewCache::DynamicMemoryUsage() const { return memusage::DynamicUsage(cacheCoins) + cachedCoinsUsage; } -CCoinsMap::const_iterator CCoinsViewCache::FetchCoins(const uint256 &txid) const { - CCoinsMap::iterator it = cacheCoins.find(txid); +CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const { + CCoinsMap::iterator it = cacheCoins.find(outpoint); if (it != cacheCoins.end()) return it; - CCoins tmp; - if (!base->GetCoins(txid, tmp)) + Coin tmp; + if (!base->GetCoin(outpoint, tmp)) return cacheCoins.end(); - CCoinsMap::iterator ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())).first; - tmp.swap(ret->second.coins); - if (ret->second.coins.IsPruned()) { - // The parent only has an empty entry for this txid; we can consider our + CCoinsMap::iterator ret = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::forward_as_tuple(std::move(tmp))).first; + if (ret->second.coin.IsSpent()) { + // The parent only has an empty entry for this outpoint; we can consider our // version as fresh. ret->second.flags = CCoinsCacheEntry::FRESH; } - cachedCoinsUsage += ret->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage += ret->second.coin.DynamicMemoryUsage(); return ret; } -bool CCoinsViewCache::GetCoins(const uint256 &txid, CCoins &coins) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +bool CCoinsViewCache::GetCoin(const COutPoint &outpoint, Coin &coin) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it != cacheCoins.end()) { - coins = it->second.coins; + coin = it->second.coin; return true; } return false; } -CCoinsModifier CCoinsViewCache::ModifyCoins(const uint256 &txid) { - assert(!hasModifier); - std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - size_t cachedCoinUsage = 0; - if (ret.second) { - if (!base->GetCoins(txid, ret.first->second.coins)) { - // The parent view does not have this entry; mark it as fresh. - ret.first->second.coins.Clear(); - ret.first->second.flags = CCoinsCacheEntry::FRESH; - } else if (ret.first->second.coins.IsPruned()) { - // The parent view only has a pruned entry for this; mark it as fresh. - ret.first->second.flags = CCoinsCacheEntry::FRESH; +void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possible_overwrite) { + assert(!coin.IsSpent()); + if (coin.out.scriptPubKey.IsUnspendable()) return; + CCoinsMap::iterator it; + bool inserted; + std::tie(it, inserted) = cacheCoins.emplace(std::piecewise_construct, std::forward_as_tuple(outpoint), std::tuple<>()); + bool fresh = false; + if (!inserted) { + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + } + if (!possible_overwrite) { + if (!it->second.coin.IsSpent()) { + throw std::logic_error("Adding new coin that replaces non-pruned entry"); } - } else { - cachedCoinUsage = ret.first->second.coins.DynamicMemoryUsage(); + fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); + } + it->second.coin = std::move(coin); + it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); + cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); +} + +void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight) { + bool fCoinbase = tx.IsCoinBase(); + const uint256& txid = tx.GetHash(); + for (size_t i = 0; i < tx.vout.size(); ++i) { + // Pass fCoinbase as the possible_overwrite flag to AddCoin, in order to correctly + // deal with the pre-BIP30 occurrances of duplicate coinbase transactions. + cache.AddCoin(COutPoint(txid, i), Coin(tx.vout[i], nHeight, fCoinbase), fCoinbase); } - // Assume that whenever ModifyCoins is called, the entry will be modified. - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, cachedCoinUsage); } -/* ModifyNewCoins allows for faster coin modification when creating the new - * outputs from a transaction. It assumes that BIP 30 (no duplicate txids) - * applies and has already been tested for (or the test is not required due to - * BIP 34, height in coinbase). If we can assume BIP 30 then we know that any - * non-coinbase transaction we are adding to the UTXO must not already exist in - * the utxo unless it is fully spent. Thus we can check only if it exists DIRTY - * at the current level of the cache, in which case it is not safe to mark it - * FRESH (b/c then its spentness still needs to flushed). If it's not dirty and - * doesn't exist or is pruned in the current cache, we know it either doesn't - * exist or is pruned in parent caches, which is the definition of FRESH. The - * exception to this is the two historical violations of BIP 30 in the chain, - * both of which were coinbases. We do not mark these fresh so we we can ensure - * that they will still be properly overwritten when spent. - */ -CCoinsModifier CCoinsViewCache::ModifyNewCoins(const uint256 &txid, bool coinbase) { - assert(!hasModifier); - std::pair<CCoinsMap::iterator, bool> ret = cacheCoins.insert(std::make_pair(txid, CCoinsCacheEntry())); - if (!coinbase) { - // New coins must not already exist. - if (!ret.first->second.coins.IsPruned()) - throw std::logic_error("ModifyNewCoins should not find pre-existing coins on a non-coinbase unless they are pruned!"); - - if (!(ret.first->second.flags & CCoinsCacheEntry::DIRTY)) { - // If the coin is known to be pruned (have no unspent outputs) in - // the current view and the cache entry is not dirty, we know the - // coin also must be pruned in the parent view as well, so it is safe - // to mark this fresh. - ret.first->second.flags |= CCoinsCacheEntry::FRESH; - } +void CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { + CCoinsMap::iterator it = FetchCoin(outpoint); + if (it == cacheCoins.end()) return; + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + if (moveout) { + *moveout = std::move(it->second.coin); + } + if (it->second.flags & CCoinsCacheEntry::FRESH) { + cacheCoins.erase(it); + } else { + it->second.flags |= CCoinsCacheEntry::DIRTY; + it->second.coin.Clear(); } - ret.first->second.coins.Clear(); - ret.first->second.flags |= CCoinsCacheEntry::DIRTY; - return CCoinsModifier(*this, ret.first, 0); } -const CCoins* CCoinsViewCache::AccessCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); +static const Coin coinEmpty; + +const Coin& CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) { - return NULL; + return coinEmpty; } else { - return &it->second.coins; + return it->second.coin; } } -bool CCoinsViewCache::HaveCoins(const uint256 &txid) const { - CCoinsMap::const_iterator it = FetchCoins(txid); - // We're using vtx.empty() instead of IsPruned here for performance reasons, - // as we only care about the case where a transaction was replaced entirely - // in a reorganization (which wipes vout entirely, as opposed to spending - // which just cleans individual outputs). - return (it != cacheCoins.end() && !it->second.coins.vout.empty()); +bool CCoinsViewCache::HaveCoin(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = FetchCoin(outpoint); + return (it != cacheCoins.end() && !it->second.coin.IsSpent()); } -bool CCoinsViewCache::HaveCoinsInCache(const uint256 &txid) const { - CCoinsMap::const_iterator it = cacheCoins.find(txid); +bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const { + CCoinsMap::const_iterator it = cacheCoins.find(outpoint); return it != cacheCoins.end(); } @@ -186,19 +138,18 @@ void CCoinsViewCache::SetBestBlock(const uint256 &hashBlockIn) { } bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn) { - assert(!hasModifier); for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Ignore non-dirty entries (optimization). CCoinsMap::iterator itUs = cacheCoins.find(it->first); if (itUs == cacheCoins.end()) { // The parent cache does not have an entry, while the child does // We can ignore it if it's both FRESH and pruned in the child - if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coins.IsPruned())) { + if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { // Otherwise we will need to create it in the parent // and move the data up and mark it as dirty CCoinsCacheEntry& entry = cacheCoins[it->first]; - entry.coins.swap(it->second.coins); - cachedCoinsUsage += entry.coins.DynamicMemoryUsage(); + entry.coin = std::move(it->second.coin); + cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); entry.flags = CCoinsCacheEntry::DIRTY; // We can mark it FRESH in the parent if it was FRESH in the child // Otherwise it might have just been flushed from the parent's cache @@ -211,21 +162,21 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn // parent cache entry has unspent outputs. If this ever happens, // it means the FRESH flag was misapplied and there is a logic // error in the calling code. - if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coins.IsPruned()) + if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); // Found the entry in the parent cache - if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { + if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { // The grandparent does not have an entry, and the child is // modified and being pruned. This means we can just delete // it from the parent. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { // A normal modification. - cachedCoinsUsage -= itUs->second.coins.DynamicMemoryUsage(); - itUs->second.coins.swap(it->second.coins); - cachedCoinsUsage += itUs->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); + itUs->second.coin = std::move(it->second.coin); + cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; // NOTE: It is possible the child has a FRESH flag here in // the event the entry we found in the parent is pruned. But @@ -249,11 +200,11 @@ bool CCoinsViewCache::Flush() { return fOk; } -void CCoinsViewCache::Uncache(const uint256& hash) +void CCoinsViewCache::Uncache(const COutPoint& hash) { CCoinsMap::iterator it = cacheCoins.find(hash); if (it != cacheCoins.end() && it->second.flags == 0) { - cachedCoinsUsage -= it->second.coins.DynamicMemoryUsage(); + cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); cacheCoins.erase(it); } } @@ -262,13 +213,6 @@ unsigned int CCoinsViewCache::GetCacheSize() const { return cacheCoins.size(); } -const CTxOut &CCoinsViewCache::GetOutputFor(const CTxIn& input) const -{ - const CCoins* coins = AccessCoins(input.prevout.hash); - assert(coins && coins->IsAvailable(input.prevout.n)); - return coins->vout[input.prevout.n]; -} - CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const { if (tx.IsCoinBase()) @@ -276,7 +220,7 @@ CAmount CCoinsViewCache::GetValueIn(const CTransaction& tx) const CAmount nResult = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) - nResult += GetOutputFor(tx.vin[i]).nValue; + nResult += AccessCoin(tx.vin[i].prevout).out.nValue; return nResult; } @@ -285,9 +229,7 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const { if (!tx.IsCoinBase()) { for (unsigned int i = 0; i < tx.vin.size(); i++) { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = AccessCoins(prevout.hash); - if (!coins || !coins->IsAvailable(prevout.n)) { + if (!HaveCoin(tx.vin[i].prevout)) { return false; } } @@ -295,25 +237,15 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const return true; } -CCoinsModifier::CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage) : cache(cache_), it(it_), cachedCoinUsage(usage) { - assert(!cache.hasModifier); - cache.hasModifier = true; -} +static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h. -CCoinsModifier::~CCoinsModifier() +const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid) { - assert(cache.hasModifier); - cache.hasModifier = false; - it->second.coins.Cleanup(); - cache.cachedCoinsUsage -= cachedCoinUsage; // Subtract the old usage - if ((it->second.flags & CCoinsCacheEntry::FRESH) && it->second.coins.IsPruned()) { - cache.cacheCoins.erase(it); - } else { - // If the coin still exists after the modification, add the new usage - cache.cachedCoinsUsage += it->second.coins.DynamicMemoryUsage(); + COutPoint iter(txid, 0); + while (iter.n < MAX_OUTPUTS_PER_BLOCK) { + const Coin& alternate = view.AccessCoin(iter); + if (!alternate.IsSpent()) return alternate; + ++iter.n; } -} - -CCoinsViewCursor::~CCoinsViewCursor() -{ + return coinEmpty; } diff --git a/src/coins.h b/src/coins.h index 8ee49b33ae..476db8f37c 100644 --- a/src/coins.h +++ b/src/coins.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_COINS_H #define BITCOIN_COINS_H +#include "primitives/transaction.h" #include "compressor.h" #include "core_memusage.h" #include "hash.h" @@ -17,137 +18,39 @@ #include <stdint.h> #include <boost/foreach.hpp> -#include <boost/unordered_map.hpp> +#include <unordered_map> -/** - * Pruned version of CTransaction: only retains metadata and unspent transaction outputs +/** + * A UTXO entry. * * Serialized format: - * - VARINT(nVersion) - * - VARINT(nCode) - * - unspentness bitvector, for vout[2] and further; least significant byte first - * - the non-spent CTxOuts (via CTxOutCompressor) - * - VARINT(nHeight) - * - * The nCode value consists of: - * - bit 0: IsCoinBase() - * - bit 1: vout[0] is not spent - * - bit 2: vout[1] is not spent - * - The higher bits encode N, the number of non-zero bytes in the following bitvector. - * - In case both bit 1 and bit 2 are unset, they encode N-1, as there must be at - * least one non-spent output). - * - * Example: 0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e - * <><><--------------------------------------------><----> - * | \ | / - * version code vout[1] height - * - * - version = 1 - * - code = 4 (vout[1] is not spent, and 0 non-zero bytes of bitvector follow) - * - unspentness bitvector: as 0 non-zero bytes follow, it has length 0 - * - vout[1]: 835800816115944e077fe7c803cfa57f29b36bf87c1d35 - * * 8358: compact amount representation for 60000000000 (600 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 816115944e077fe7c803cfa57f29b36bf87c1d35: address uint160 - * - height = 203998 - * - * - * Example: 0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b - * <><><--><--------------------------------------------------><----------------------------------------------><----> - * / \ \ | | / - * version code unspentness vout[4] vout[16] height - * - * - version = 1 - * - code = 9 (coinbase, neither vout[0] or vout[1] are unspent, - * 2 (1, +1 because both bit 1 and bit 2 are unset) non-zero bitvector bytes follow) - * - unspentness bitvector: bits 2 (0x04) and 14 (0x4000) are set, so vout[2+2] and vout[14+2] are unspent - * - vout[4]: 86ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4ee - * * 86ef97d579: compact amount representation for 234925952 (2.35 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 61b01caab50f1b8e9c50a5057eb43c2d9563a4ee: address uint160 - * - vout[16]: bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4 - * * bbd123: compact amount representation for 110397 (0.001 BTC) - * * 00: special txout type pay-to-pubkey-hash - * * 8c988f1a4a4de2161e0f50aac7f17e7f9555caa4: address uint160 - * - height = 120891 + * - VARINT((coinbase ? 1 : 0) | (height << 1)) + * - the non-spent CTxOut (via CTxOutCompressor) */ -class CCoins +class Coin { public: - //! whether transaction is a coinbase - bool fCoinBase; - - //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped - std::vector<CTxOut> vout; + //! unspent transaction output + CTxOut out; - //! at which height this transaction was included in the active block chain - int nHeight; + //! whether containing transaction was a coinbase + unsigned int fCoinBase : 1; - //! version of the CTransaction; accesses to this value should probably check for nHeight as well, - //! as new tx version will probably only be introduced at certain heights - int nVersion; + //! at which height this containing transaction was included in the active block chain + uint32_t nHeight : 31; - void FromTx(const CTransaction &tx, int nHeightIn) { - fCoinBase = tx.IsCoinBase(); - vout = tx.vout; - nHeight = nHeightIn; - nVersion = tx.nVersion; - ClearUnspendable(); - } - - //! construct a CCoins from a CTransaction, at a given height - CCoins(const CTransaction &tx, int nHeightIn) { - FromTx(tx, nHeightIn); - } + //! construct a Coin from a CTxOut and height/coinbase information. + Coin(CTxOut&& outIn, int nHeightIn, bool fCoinBaseIn) : out(std::move(outIn)), fCoinBase(fCoinBaseIn), nHeight(nHeightIn) {} + Coin(const CTxOut& outIn, int nHeightIn, bool fCoinBaseIn) : out(outIn), fCoinBase(fCoinBaseIn),nHeight(nHeightIn) {} void Clear() { + out.SetNull(); fCoinBase = false; - std::vector<CTxOut>().swap(vout); nHeight = 0; - nVersion = 0; } //! empty constructor - CCoins() : fCoinBase(false), vout(0), nHeight(0), nVersion(0) { } - - //!remove spent outputs at the end of vout - void Cleanup() { - while (vout.size() > 0 && vout.back().IsNull()) - vout.pop_back(); - if (vout.empty()) - std::vector<CTxOut>().swap(vout); - } - - void ClearUnspendable() { - BOOST_FOREACH(CTxOut &txout, vout) { - if (txout.scriptPubKey.IsUnspendable()) - txout.SetNull(); - } - Cleanup(); - } - - void swap(CCoins &to) { - std::swap(to.fCoinBase, fCoinBase); - to.vout.swap(vout); - std::swap(to.nHeight, nHeight); - std::swap(to.nVersion, nVersion); - } - - //! equality test - friend bool operator==(const CCoins &a, const CCoins &b) { - // Empty CCoins objects are always equal. - if (a.IsPruned() && b.IsPruned()) - return true; - return a.fCoinBase == b.fCoinBase && - a.nHeight == b.nHeight && - a.nVersion == b.nVersion && - a.vout == b.vout; - } - friend bool operator!=(const CCoins &a, const CCoins &b) { - return !(a == b); - } - - void CalcMaskSize(unsigned int &nBytes, unsigned int &nNonzeroBytes) const; + Coin() : fCoinBase(false), nHeight(0) { } bool IsCoinBase() const { return fCoinBase; @@ -155,115 +58,52 @@ public: template<typename Stream> void Serialize(Stream &s) const { - unsigned int nMaskSize = 0, nMaskCode = 0; - CalcMaskSize(nMaskSize, nMaskCode); - bool fFirst = vout.size() > 0 && !vout[0].IsNull(); - bool fSecond = vout.size() > 1 && !vout[1].IsNull(); - assert(fFirst || fSecond || nMaskCode); - unsigned int nCode = 8*(nMaskCode - (fFirst || fSecond ? 0 : 1)) + (fCoinBase ? 1 : 0) + (fFirst ? 2 : 0) + (fSecond ? 4 : 0); - // version - ::Serialize(s, VARINT(this->nVersion)); - // header code - ::Serialize(s, VARINT(nCode)); - // spentness bitmask - for (unsigned int b = 0; b<nMaskSize; b++) { - unsigned char chAvail = 0; - for (unsigned int i = 0; i < 8 && 2+b*8+i < vout.size(); i++) - if (!vout[2+b*8+i].IsNull()) - chAvail |= (1 << i); - ::Serialize(s, chAvail); - } - // txouts themself - for (unsigned int i = 0; i < vout.size(); i++) { - if (!vout[i].IsNull()) - ::Serialize(s, CTxOutCompressor(REF(vout[i]))); - } - // coinbase height - ::Serialize(s, VARINT(nHeight)); + assert(!IsSpent()); + uint32_t code = nHeight * 2 + fCoinBase; + ::Serialize(s, VARINT(code)); + ::Serialize(s, CTxOutCompressor(REF(out))); } template<typename Stream> void Unserialize(Stream &s) { - unsigned int nCode = 0; - // version - ::Unserialize(s, VARINT(this->nVersion)); - // header code - ::Unserialize(s, VARINT(nCode)); - fCoinBase = nCode & 1; - std::vector<bool> vAvail(2, false); - vAvail[0] = (nCode & 2) != 0; - vAvail[1] = (nCode & 4) != 0; - unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); - // spentness bitmask - while (nMaskCode > 0) { - unsigned char chAvail = 0; - ::Unserialize(s, chAvail); - for (unsigned int p = 0; p < 8; p++) { - bool f = (chAvail & (1 << p)) != 0; - vAvail.push_back(f); - } - if (chAvail != 0) - nMaskCode--; - } - // txouts themself - vout.assign(vAvail.size(), CTxOut()); - for (unsigned int i = 0; i < vAvail.size(); i++) { - if (vAvail[i]) - ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); - } - // coinbase height - ::Unserialize(s, VARINT(nHeight)); - Cleanup(); + uint32_t code = 0; + ::Unserialize(s, VARINT(code)); + nHeight = code >> 1; + fCoinBase = code & 1; + ::Unserialize(s, REF(CTxOutCompressor(out))); } - //! mark a vout spent - bool Spend(uint32_t nPos); - - //! check whether a particular output is still available - bool IsAvailable(unsigned int nPos) const { - return (nPos < vout.size() && !vout[nPos].IsNull()); - } - - //! check whether the entire CCoins is spent - //! note that only !IsPruned() CCoins can be serialized - bool IsPruned() const { - BOOST_FOREACH(const CTxOut &out, vout) - if (!out.IsNull()) - return false; - return true; + bool IsSpent() const { + return out.IsNull(); } size_t DynamicMemoryUsage() const { - size_t ret = memusage::DynamicUsage(vout); - BOOST_FOREACH(const CTxOut &out, vout) { - ret += RecursiveDynamicUsage(out.scriptPubKey); - } - return ret; + return memusage::DynamicUsage(out.scriptPubKey); } }; -class SaltedTxidHasher +class SaltedOutpointHasher { private: /** Salt */ const uint64_t k0, k1; public: - SaltedTxidHasher(); + SaltedOutpointHasher(); /** * This *must* return size_t. With Boost 1.46 on 32-bit systems the * unordered_map will behave unpredictably if the custom hasher returns a * uint64_t, resulting in failures when syncing the chain (#4634). */ - size_t operator()(const uint256& txid) const { - return SipHashUint256(k0, k1, txid); + size_t operator()(const COutPoint& id) const { + return SipHashUint256Extra(k0, k1, id.hash, id.n); } }; struct CCoinsCacheEntry { - CCoins coins; // The actual cached data. + Coin coin; // The actual cached data. unsigned char flags; enum Flags { @@ -276,20 +116,21 @@ struct CCoinsCacheEntry */ }; - CCoinsCacheEntry() : coins(), flags(0) {} + CCoinsCacheEntry() : flags(0) {} + explicit CCoinsCacheEntry(Coin&& coin_) : coin(std::move(coin_)), flags(0) {} }; -typedef boost::unordered_map<uint256, CCoinsCacheEntry, SaltedTxidHasher> CCoinsMap; +typedef std::unordered_map<COutPoint, CCoinsCacheEntry, SaltedOutpointHasher> CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor { public: CCoinsViewCursor(const uint256 &hashBlockIn): hashBlock(hashBlockIn) {} - virtual ~CCoinsViewCursor(); + virtual ~CCoinsViewCursor() {} - virtual bool GetKey(uint256 &key) const = 0; - virtual bool GetValue(CCoins &coins) const = 0; + virtual bool GetKey(COutPoint &key) const = 0; + virtual bool GetValue(Coin &coin) const = 0; virtual unsigned int GetValueSize() const = 0; virtual bool Valid() const = 0; @@ -305,17 +146,17 @@ private: class CCoinsView { public: - //! Retrieve the CCoins (unspent transaction outputs) for a given txid - virtual bool GetCoins(const uint256 &txid, CCoins &coins) const; + //! Retrieve the Coin (unspent transaction output) for a given outpoint. + virtual bool GetCoin(const COutPoint &outpoint, Coin &coin) const; - //! Just check whether we have data for a given txid. - //! This may (but cannot always) return true for fully spent transactions - virtual bool HaveCoins(const uint256 &txid) const; + //! Just check whether we have data for a given outpoint. + //! This may (but cannot always) return true for spent outputs. + virtual bool HaveCoin(const COutPoint &outpoint) const; //! Retrieve the block hash whose state this CCoinsView currently represents virtual uint256 GetBestBlock() const; - //! Do a bulk modification (multiple CCoins changes + BestBlock change). + //! Do a bulk modification (multiple Coin changes + BestBlock change). //! The passed mapCoins can be modified. virtual bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); @@ -324,6 +165,9 @@ public: //! As we use CCoinsViews polymorphically, have a virtual destructor virtual ~CCoinsView() {} + + //! Estimate database size (0 if not implemented) + virtual size_t EstimateSize() const { return 0; } }; @@ -335,45 +179,20 @@ protected: public: CCoinsViewBacked(CCoinsView *viewIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; void SetBackend(CCoinsView &viewIn); - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; + size_t EstimateSize() const override; }; -class CCoinsViewCache; - -/** - * A reference to a mutable cache entry. Encapsulating it allows us to run - * cleanup code after the modification is finished, and keeping track of - * concurrent modifications. - */ -class CCoinsModifier -{ -private: - CCoinsViewCache& cache; - CCoinsMap::iterator it; - size_t cachedCoinUsage; // Cached memory usage of the CCoins object before modification - CCoinsModifier(CCoinsViewCache& cache_, CCoinsMap::iterator it_, size_t usage); - -public: - CCoins* operator->() { return &it->second.coins; } - CCoins& operator*() { return it->second.coins; } - ~CCoinsModifier(); - friend class CCoinsViewCache; -}; - /** CCoinsView that adds a memory cache for transactions to another CCoinsView */ class CCoinsViewCache : public CCoinsViewBacked { protected: - /* Whether this cache has an active modifier. */ - bool hasModifier; - - /** * Make mutable so that we can "fill the cache" even from Get-methods * declared as "const". @@ -381,51 +200,45 @@ protected: mutable uint256 hashBlock; mutable CCoinsMap cacheCoins; - /* Cached dynamic memory usage for the inner CCoins objects. */ + /* Cached dynamic memory usage for the inner Coin objects. */ mutable size_t cachedCoinsUsage; public: CCoinsViewCache(CCoinsView *baseIn); - ~CCoinsViewCache(); // Standard CCoinsView methods - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; uint256 GetBestBlock() const; void SetBestBlock(const uint256 &hashBlock); bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); /** - * Check if we have the given tx already loaded in this cache. - * The semantics are the same as HaveCoins(), but no calls to + * Check if we have the given utxo already loaded in this cache. + * The semantics are the same as HaveCoin(), but no calls to * the backing CCoinsView are made. */ - bool HaveCoinsInCache(const uint256 &txid) const; + bool HaveCoinInCache(const COutPoint &outpoint) const; /** - * Return a pointer to CCoins in the cache, or NULL if not found. This is - * more efficient than GetCoins. Modifications to other cache entries are + * Return a reference to Coin in the cache, or a pruned one if not found. This is + * more efficient than GetCoin. Modifications to other cache entries are * allowed while accessing the returned pointer. */ - const CCoins* AccessCoins(const uint256 &txid) const; + const Coin& AccessCoin(const COutPoint &output) const; /** - * Return a modifiable reference to a CCoins. If no entry with the given - * txid exists, a new one is created. Simultaneous modifications are not - * allowed. + * Add a coin. Set potential_overwrite to true if a non-pruned version may + * already exist. */ - CCoinsModifier ModifyCoins(const uint256 &txid); + void AddCoin(const COutPoint& outpoint, Coin&& coin, bool potential_overwrite); /** - * Return a modifiable reference to a CCoins. Assumes that no entry with the given - * txid exists and creates a new one. This saves a database access in the case where - * the coins were to be wiped out by FromTx anyway. This should not be called with - * the 2 historical coinbase duplicate pairs because the new coins are marked fresh, and - * in the event the duplicate coinbase was spent before a flush, the now pruned coins - * would not properly overwrite the first coinbase of the pair. Simultaneous modifications - * are not allowed. + * Spend a coin. Pass moveto in order to get the deleted data. + * If no unspent output exists for the passed outpoint, this call + * has no effect. */ - CCoinsModifier ModifyNewCoins(const uint256 &txid, bool coinbase); + void SpendCoin(const COutPoint &outpoint, Coin* moveto = nullptr); /** * Push the modifications applied to this cache to its base. @@ -435,12 +248,12 @@ public: bool Flush(); /** - * Removes the transaction with the given hash from the cache, if it is + * Removes the UTXO with the given outpoint from the cache, if it is * not modified. */ - void Uncache(const uint256 &txid); + void Uncache(const COutPoint &outpoint); - //! Calculate the size of the cache (in number of transactions) + //! Calculate the size of the cache (in number of transaction outputs) unsigned int GetCacheSize() const; //! Calculate the size of the cache (in bytes) @@ -459,12 +272,8 @@ public: //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; - const CTxOut &GetOutputFor(const CTxIn& input) const; - - friend class CCoinsModifier; - private: - CCoinsMap::const_iterator FetchCoins(const uint256 &txid) const; + CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; /** * By making the copy constructor private, we prevent accidentally using it when one intends to create a cache on top of a base cache. @@ -472,4 +281,13 @@ private: CCoinsViewCache(const CCoinsViewCache &); }; +//! Utility function to add all of a transaction's outputs to a cache. +// It assumes that overwrites are only possible for coinbase transactions, +// TODO: pass in a boolean to limit these possible overwrites to known +// (pre-BIP34) cases. +void AddCoins(CCoinsViewCache& cache, const CTransaction& tx, int nHeight); + +//! Utility function to find any unspent output with a given txid. +const Coin& AccessByTxid(const CCoinsViewCache& cache, const uint256& txid); + #endif // BITCOIN_COINS_H diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp new file mode 100644 index 0000000000..bf68f8754b --- /dev/null +++ b/src/consensus/tx_verify.cpp @@ -0,0 +1,246 @@ +// Copyright (c) 2017-2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "tx_verify.h" + +#include "consensus.h" +#include "primitives/transaction.h" +#include "script/interpreter.h" +#include "validation.h" + +// TODO remove the following dependencies +#include "chain.h" +#include "coins.h" +#include "utilmoneystr.h" + +bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) +{ + if (tx.nLockTime == 0) + return true; + if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) + return true; + for (const auto& txin : tx.vin) { + if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) + return false; + } + return true; +} + +std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) +{ + assert(prevHeights->size() == tx.vin.size()); + + // Will be set to the equivalent height- and time-based nLockTime + // values that would be necessary to satisfy all relative lock- + // time constraints given our view of block chain history. + // The semantics of nLockTime are the last invalid height/time, so + // use -1 to have the effect of any height or time being valid. + int nMinHeight = -1; + int64_t nMinTime = -1; + + // tx.nVersion is signed integer so requires cast to unsigned otherwise + // we would be doing a signed comparison and half the range of nVersion + // wouldn't support BIP 68. + bool fEnforceBIP68 = static_cast<uint32_t>(tx.nVersion) >= 2 + && flags & LOCKTIME_VERIFY_SEQUENCE; + + // Do not enforce sequence numbers as a relative lock time + // unless we have been instructed to + if (!fEnforceBIP68) { + return std::make_pair(nMinHeight, nMinTime); + } + + for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { + const CTxIn& txin = tx.vin[txinIndex]; + + // Sequence numbers with the most significant bit set are not + // treated as relative lock-times, nor are they given any + // consensus-enforced meaning at this point. + if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_DISABLE_FLAG) { + // The height of this input is not relevant for sequence locks + (*prevHeights)[txinIndex] = 0; + continue; + } + + int nCoinHeight = (*prevHeights)[txinIndex]; + + if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) { + int64_t nCoinTime = block.GetAncestor(std::max(nCoinHeight-1, 0))->GetMedianTimePast(); + // NOTE: Subtract 1 to maintain nLockTime semantics + // BIP 68 relative lock times have the semantics of calculating + // the first block or time at which the transaction would be + // valid. When calculating the effective block time or height + // for the entire transaction, we switch to using the + // semantics of nLockTime which is the last invalid block + // time or height. Thus we subtract 1 from the calculated + // time or height. + + // Time-based relative lock-times are measured from the + // smallest allowed timestamp of the block containing the + // txout being spent, which is the median time past of the + // block prior. + nMinTime = std::max(nMinTime, nCoinTime + (int64_t)((txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) << CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) - 1); + } else { + nMinHeight = std::max(nMinHeight, nCoinHeight + (int)(txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) - 1); + } + } + + return std::make_pair(nMinHeight, nMinTime); +} + +bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair) +{ + assert(block.pprev); + int64_t nBlockTime = block.pprev->GetMedianTimePast(); + if (lockPair.first >= block.nHeight || lockPair.second >= nBlockTime) + return false; + + return true; +} + +bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) +{ + return EvaluateSequenceLocks(block, CalculateSequenceLocks(tx, flags, prevHeights, block)); +} + +unsigned int GetLegacySigOpCount(const CTransaction& tx) +{ + unsigned int nSigOps = 0; + for (const auto& txin : tx.vin) + { + nSigOps += txin.scriptSig.GetSigOpCount(false); + } + for (const auto& txout : tx.vout) + { + nSigOps += txout.scriptPubKey.GetSigOpCount(false); + } + return nSigOps; +} + +unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& inputs) +{ + if (tx.IsCoinBase()) + return 0; + + unsigned int nSigOps = 0; + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; + if (prevout.scriptPubKey.IsPayToScriptHash()) + nSigOps += prevout.scriptPubKey.GetSigOpCount(tx.vin[i].scriptSig); + } + return nSigOps; +} + +int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags) +{ + int64_t nSigOps = GetLegacySigOpCount(tx) * WITNESS_SCALE_FACTOR; + + if (tx.IsCoinBase()) + return nSigOps; + + if (flags & SCRIPT_VERIFY_P2SH) { + nSigOps += GetP2SHSigOpCount(tx, inputs) * WITNESS_SCALE_FACTOR; + } + + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const CTxOut &prevout = inputs.AccessCoin(tx.vin[i].prevout).out; + nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); + } + return nSigOps; +} + +bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) +{ + // Basic checks that don't depend on any context + if (tx.vin.empty()) + return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); + if (tx.vout.empty()) + return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); + // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) + if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); + + // Check for negative or overflow output values + CAmount nValueOut = 0; + for (const auto& txout : tx.vout) + { + if (txout.nValue < 0) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); + if (txout.nValue > MAX_MONEY) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-toolarge"); + nValueOut += txout.nValue; + if (!MoneyRange(nValueOut)) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); + } + + // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock + if (fCheckDuplicateInputs) { + std::set<COutPoint> vInOutPoints; + for (const auto& txin : tx.vin) + { + if (!vInOutPoints.insert(txin.prevout).second) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); + } + } + + if (tx.IsCoinBase()) + { + if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) + return state.DoS(100, false, REJECT_INVALID, "bad-cb-length"); + } + else + { + for (const auto& txin : tx.vin) + if (txin.prevout.IsNull()) + return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); + } + + return true; +} + +bool Consensus::CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight) +{ + // This doesn't trigger the DoS code on purpose; if it did, it would make it easier + // for an attacker to attempt to split the network. + if (!inputs.HaveInputs(tx)) + return state.Invalid(false, 0, "", "Inputs unavailable"); + + CAmount nValueIn = 0; + CAmount nFees = 0; + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const COutPoint &prevout = tx.vin[i].prevout; + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); + + // If prev is coinbase, check that it's matured + if (coin.IsCoinBase()) { + if (nSpendHeight - coin.nHeight < COINBASE_MATURITY) + return state.Invalid(false, + REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", + strprintf("tried to spend coinbase at depth %d", nSpendHeight - coin.nHeight)); + } + + // Check for negative or overflow input values + nValueIn += coin.out.nValue; + if (!MoneyRange(coin.out.nValue) || !MoneyRange(nValueIn)) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); + + } + + if (nValueIn < tx.GetValueOut()) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, + strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(tx.GetValueOut()))); + + // Tally transaction fees + CAmount nTxFee = nValueIn - tx.GetValueOut(); + if (nTxFee < 0) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-negative"); + nFees += nTxFee; + if (!MoneyRange(nFees)) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); + return true; +} diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h new file mode 100644 index 0000000000..d46d3294ca --- /dev/null +++ b/src/consensus/tx_verify.h @@ -0,0 +1,78 @@ +// Copyright (c) 2017-2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CONSENSUS_TX_VERIFY_H +#define BITCOIN_CONSENSUS_TX_VERIFY_H + +#include <stdint.h> +#include <vector> + +class CBlockIndex; +class CCoinsViewCache; +class CTransaction; +class CValidationState; + +/** Transaction validation functions */ + +/** Context-independent validity checks */ +bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); + +namespace Consensus { +/** + * Check whether all inputs of this transaction are valid (no double spends and amounts) + * This does not modify the UTXO set. This does not check scripts and sigs. + * Preconditions: tx.IsCoinBase() is false. + */ +bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight); +} // namespace Consensus + +/** Auxiliary functions for transaction validation (ideally should not be exposed) */ + +/** + * Count ECDSA signature operations the old-fashioned (pre-0.6) way + * @return number of sigops this transaction's outputs will produce when spent + * @see CTransaction::FetchInputs + */ +unsigned int GetLegacySigOpCount(const CTransaction& tx); + +/** + * Count ECDSA signature operations in pay-to-script-hash inputs. + * + * @param[in] mapInputs Map of previous transactions that have outputs we're spending + * @return maximum number of sigops required to validate this transaction's inputs + * @see CTransaction::FetchInputs + */ +unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& mapInputs); + +/** + * Compute total signature operation cost of a transaction. + * @param[in] tx Transaction for which we are computing the cost + * @param[in] inputs Map of previous transactions that have outputs we're spending + * @param[out] flags Script verification flags + * @return Total signature operation cost of tx + */ +int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags); + +/** + * Check if transaction is final and can be included in a block with the + * specified height and time. Consensus critical. + */ +bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime); + +/** + * Calculates the block height and previous block's median time past at + * which the transaction will be considered final in the context of BIP 68. + * Also removes from the vector of input heights any entries which did not + * correspond to sequence locked inputs as they do not affect the calculation. + */ +std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block); + +bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair); +/** + * Check if transaction is final per BIP 68 sequence numbers and can be included in a block. + * Consensus critical. Takes as input a list of heights at which tx's inputs (in order) confirmed. + */ +bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block); + +#endif // BITCOIN_CONSENSUS_TX_VERIFY_H diff --git a/src/core_memusage.h b/src/core_memusage.h index 5e10182075..e4ccd54c42 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -63,4 +63,9 @@ static inline size_t RecursiveDynamicUsage(const CBlockLocator& locator) { return memusage::DynamicUsage(locator.vHave); } +template<typename X> +static inline size_t RecursiveDynamicUsage(const std::shared_ptr<X>& p) { + return p ? memusage::DynamicUsage(p) + RecursiveDynamicUsage(*p) : 0; +} + #endif // BITCOIN_CORE_MEMUSAGE_H diff --git a/src/core_write.cpp b/src/core_write.cpp index a3ca87c8b5..d116e617ee 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -151,6 +151,8 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry) entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); entry.pushKV("version", tx.nVersion); + entry.pushKV("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)); + entry.pushKV("vsize", (GetTransactionWeight(tx) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR); entry.pushKV("locktime", (int64_t)tx.nLockTime); UniValue vin(UniValue::VARR); diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp new file mode 100644 index 0000000000..816ae870e1 --- /dev/null +++ b/src/crypto/chacha20.cpp @@ -0,0 +1,180 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Based on the public domain implementation 'merged' by D. J. Bernstein +// See https://cr.yp.to/chacha.html. + +#include "crypto/common.h" +#include "crypto/chacha20.h" + +#include <string.h> + +constexpr static inline uint32_t rotl32(uint32_t v, int c) { return (v << c) | (v >> (32 - c)); } + +#define QUARTERROUND(a,b,c,d) \ + a += b; d = rotl32(d ^ a, 16); \ + c += d; b = rotl32(b ^ c, 12); \ + a += b; d = rotl32(d ^ a, 8); \ + c += d; b = rotl32(b ^ c, 7); + +static const unsigned char sigma[] = "expand 32-byte k"; +static const unsigned char tau[] = "expand 16-byte k"; + +void ChaCha20::SetKey(const unsigned char* k, size_t keylen) +{ + const unsigned char *constants; + + input[4] = ReadLE32(k + 0); + input[5] = ReadLE32(k + 4); + input[6] = ReadLE32(k + 8); + input[7] = ReadLE32(k + 12); + if (keylen == 32) { /* recommended */ + k += 16; + constants = sigma; + } else { /* keylen == 16 */ + constants = tau; + } + input[8] = ReadLE32(k + 0); + input[9] = ReadLE32(k + 4); + input[10] = ReadLE32(k + 8); + input[11] = ReadLE32(k + 12); + input[0] = ReadLE32(constants + 0); + input[1] = ReadLE32(constants + 4); + input[2] = ReadLE32(constants + 8); + input[3] = ReadLE32(constants + 12); + input[12] = 0; + input[13] = 0; + input[14] = 0; + input[15] = 0; +} + +ChaCha20::ChaCha20() +{ + memset(input, 0, sizeof(input)); +} + +ChaCha20::ChaCha20(const unsigned char* k, size_t keylen) +{ + SetKey(k, keylen); +} + +void ChaCha20::SetIV(uint64_t iv) +{ + input[14] = iv; + input[15] = iv >> 32; +} + +void ChaCha20::Seek(uint64_t pos) +{ + input[12] = pos; + input[13] = pos >> 32; +} + +void ChaCha20::Output(unsigned char* c, size_t bytes) +{ + uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; + unsigned char *ctarget = NULL; + unsigned char tmp[64]; + unsigned int i; + + if (!bytes) return; + + j0 = input[0]; + j1 = input[1]; + j2 = input[2]; + j3 = input[3]; + j4 = input[4]; + j5 = input[5]; + j6 = input[6]; + j7 = input[7]; + j8 = input[8]; + j9 = input[9]; + j10 = input[10]; + j11 = input[11]; + j12 = input[12]; + j13 = input[13]; + j14 = input[14]; + j15 = input[15]; + + for (;;) { + if (bytes < 64) { + ctarget = c; + c = tmp; + } + x0 = j0; + x1 = j1; + x2 = j2; + x3 = j3; + x4 = j4; + x5 = j5; + x6 = j6; + x7 = j7; + x8 = j8; + x9 = j9; + x10 = j10; + x11 = j11; + x12 = j12; + x13 = j13; + x14 = j14; + x15 = j15; + for (i = 20;i > 0;i -= 2) { + QUARTERROUND( x0, x4, x8,x12) + QUARTERROUND( x1, x5, x9,x13) + QUARTERROUND( x2, x6,x10,x14) + QUARTERROUND( x3, x7,x11,x15) + QUARTERROUND( x0, x5,x10,x15) + QUARTERROUND( x1, x6,x11,x12) + QUARTERROUND( x2, x7, x8,x13) + QUARTERROUND( x3, x4, x9,x14) + } + x0 += j0; + x1 += j1; + x2 += j2; + x3 += j3; + x4 += j4; + x5 += j5; + x6 += j6; + x7 += j7; + x8 += j8; + x9 += j9; + x10 += j10; + x11 += j11; + x12 += j12; + x13 += j13; + x14 += j14; + x15 += j15; + + ++j12; + if (!j12) ++j13; + + WriteLE32(c + 0, x0); + WriteLE32(c + 4, x1); + WriteLE32(c + 8, x2); + WriteLE32(c + 12, x3); + WriteLE32(c + 16, x4); + WriteLE32(c + 20, x5); + WriteLE32(c + 24, x6); + WriteLE32(c + 28, x7); + WriteLE32(c + 32, x8); + WriteLE32(c + 36, x9); + WriteLE32(c + 40, x10); + WriteLE32(c + 44, x11); + WriteLE32(c + 48, x12); + WriteLE32(c + 52, x13); + WriteLE32(c + 56, x14); + WriteLE32(c + 60, x15); + + if (bytes <= 64) { + if (bytes < 64) { + for (i = 0;i < bytes;++i) ctarget[i] = c[i]; + } + input[12] = j12; + input[13] = j13; + return; + } + bytes -= 64; + c += 64; + } +} diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h new file mode 100644 index 0000000000..a305977bcd --- /dev/null +++ b/src/crypto/chacha20.h @@ -0,0 +1,26 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_CHACHA20_H +#define BITCOIN_CRYPTO_CHACHA20_H + +#include <stdint.h> +#include <stdlib.h> + +/** A PRNG class for ChaCha20. */ +class ChaCha20 +{ +private: + uint32_t input[16]; + +public: + ChaCha20(); + ChaCha20(const unsigned char* key, size_t keylen); + void SetKey(const unsigned char* key, size_t keylen); + void SetIV(uint64_t iv); + void Seek(uint64_t pos); + void Output(unsigned char* output, size_t bytes); +}; + +#endif // BITCOIN_CRYPTO_CHACHA20_H diff --git a/src/crypto/common.h b/src/crypto/common.h index 4a9d1150b6..bcca3d30ea 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -79,4 +79,25 @@ void static inline WriteBE64(unsigned char* ptr, uint64_t x) memcpy(ptr, (char*)&v, 8); } +/** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */ +uint64_t static inline CountBits(uint64_t x) +{ +#ifdef HAVE_DECL___BUILTIN_CLZL + if (sizeof(unsigned long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; + } +#endif +#ifdef HAVE_DECL___BUILTIN_CLZLL + if (sizeof(unsigned long long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0; + } +#endif + int ret = 0; + while (x) { + x >>= 1; + ++ret; + } + return ret; +} + #endif // BITCOIN_CRYPTO_COMMON_H diff --git a/src/dbwrapper.h b/src/dbwrapper.h index b13e98b7a4..24ef71bfbf 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -55,11 +55,19 @@ private: CDataStream ssKey; CDataStream ssValue; + size_t size_estimate; + public: /** * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION) { }; + CDBBatch(const CDBWrapper &_parent) : parent(_parent), ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { }; + + void Clear() + { + batch.Clear(); + size_estimate = 0; + } template <typename K, typename V> void Write(const K& key, const V& value) @@ -74,6 +82,14 @@ public: leveldb::Slice slValue(ssValue.data(), ssValue.size()); batch.Put(slKey, slValue); + // LevelDB serializes writes as: + // - byte: header + // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...) + // - byte[]: key + // - varint: value length + // - byte[]: value + // The formula below assumes the key and value are both less than 16k. + size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); ssKey.clear(); ssValue.clear(); } @@ -86,8 +102,16 @@ public: leveldb::Slice slKey(ssKey.data(), ssKey.size()); batch.Delete(slKey); + // LevelDB serializes erases as: + // - byte: header + // - varint: key length + // - byte[]: key + // The formula below assumes the key is less than 16kB. + size_estimate += 2 + (slKey.size() > 127) + slKey.size(); ssKey.clear(); } + + size_t SizeEstimate() const { return size_estimate; } }; class CDBIterator @@ -281,7 +305,22 @@ public: * Return true if the database managed by this class contains no entries. */ bool IsEmpty(); + + template<typename K> + size_t EstimateSize(const K& key_begin, const K& key_end) const + { + CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION); + ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey1 << key_begin; + ssKey2 << key_end; + leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + uint64_t size = 0; + leveldb::Range range(slKey1, slKey2); + pdb->GetApproximateSizes(&range, 1, &size); + return size; + } }; #endif // BITCOIN_DBWRAPPER_H - @@ -21,4 +21,4 @@ namespace fsbridge { FILE *freopen(const fs::path& p, const char *mode, FILE *stream); }; -#endif +#endif // BITCOIN_FS_H diff --git a/src/hash.cpp b/src/hash.cpp index a14a2386a2..b361c90d16 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -208,3 +208,44 @@ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val) SIPROUND; return v0 ^ v1 ^ v2 ^ v3; } + +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra) +{ + /* Specialized implementation for efficiency */ + uint64_t d = val.GetUint64(0); + + uint64_t v0 = 0x736f6d6570736575ULL ^ k0; + uint64_t v1 = 0x646f72616e646f6dULL ^ k1; + uint64_t v2 = 0x6c7967656e657261ULL ^ k0; + uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ d; + + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(1); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(2); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = val.GetUint64(3); + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + d = (((uint64_t)36) << 56) | extra; + v3 ^= d; + SIPROUND; + SIPROUND; + v0 ^= d; + v2 ^= 0xFF; + SIPROUND; + SIPROUND; + SIPROUND; + SIPROUND; + return v0 ^ v1 ^ v2 ^ v3; +} diff --git a/src/hash.h b/src/hash.h index eacb8f04fe..b9952d39fc 100644 --- a/src/hash.h +++ b/src/hash.h @@ -160,6 +160,41 @@ public: } }; +/** Reads data from an underlying stream, while hashing the read data. */ +template<typename Source> +class CHashVerifier : public CHashWriter +{ +private: + Source* source; + +public: + CHashVerifier(Source* source_) : CHashWriter(source_->GetType(), source_->GetVersion()), source(source_) {} + + void read(char* pch, size_t nSize) + { + source->read(pch, nSize); + this->write(pch, nSize); + } + + void ignore(size_t nSize) + { + char data[1024]; + while (nSize > 0) { + size_t now = std::min<size_t>(nSize, 1024); + read(data, now); + nSize -= now; + } + } + + template<typename T> + CHashVerifier<Source>& operator>>(T& obj) + { + // Unserialize from this stream + ::Unserialize(*this, obj); + return (*this); + } +}; + /** Compute the 256-bit hash of an object's serialization. */ template<typename T> uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION) @@ -206,5 +241,6 @@ public: * .Finalize() */ uint64_t SipHashUint256(uint64_t k0, uint64_t k1, const uint256& val); +uint64_t SipHashUint256Extra(uint64_t k0, uint64_t k1, const uint256& val, uint32_t extra); #endif // BITCOIN_HASH_H diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 21c64c5c83..5ab6d8d732 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -30,7 +30,7 @@ static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; class HTTPRPCTimer : public RPCTimerBase { public: - HTTPRPCTimer(struct event_base* eventBase, boost::function<void(void)>& func, int64_t millis) : + HTTPRPCTimer(struct event_base* eventBase, std::function<void(void)>& func, int64_t millis) : ev(eventBase, false, func) { struct timeval tv; @@ -52,7 +52,7 @@ public: { return "HTTP"; } - RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) + RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) { return new HTTPRPCTimer(base, func, millis); } @@ -93,9 +93,9 @@ static bool multiUserAuthorized(std::string strUserPass) std::string strUser = strUserPass.substr(0, strUserPass.find(":")); std::string strPass = strUserPass.substr(strUserPass.find(":") + 1); - if (mapMultiArgs.count("-rpcauth") > 0) { + if (gArgs.IsArgSet("-rpcauth")) { //Search for multi-user login/pass "rpcauth" from config - BOOST_FOREACH(std::string strRPCAuth, mapMultiArgs.at("-rpcauth")) + BOOST_FOREACH(std::string strRPCAuth, gArgs.GetArgs("-rpcauth")) { std::vector<std::string> vFields; boost::split(vFields, strRPCAuth, boost::is_any_of(":$")); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index e7df23295d..0d1cba3fd2 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -196,9 +196,8 @@ static bool InitHTTPAllowList() LookupHost("::1", localv6, false); rpc_allow_subnets.push_back(CSubNet(localv4, 8)); // always allow IPv4 local subnet rpc_allow_subnets.push_back(CSubNet(localv6)); // always allow IPv6 localhost - if (mapMultiArgs.count("-rpcallowip")) { - const std::vector<std::string>& vAllow = mapMultiArgs.at("-rpcallowip"); - for (std::string strAllow : vAllow) { + if (gArgs.IsArgSet("-rpcallowip")) { + for (const std::string& strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; LookupSubNet(strAllow.c_str(), subnet); if (!subnet.IsValid()) { @@ -321,12 +320,11 @@ static bool HTTPBindAddresses(struct evhttp* http) if (IsArgSet("-rpcbind")) { LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n"); } - } else if (mapMultiArgs.count("-rpcbind")) { // Specific bind address - const std::vector<std::string>& vbind = mapMultiArgs.at("-rpcbind"); - for (std::vector<std::string>::const_iterator i = vbind.begin(); i != vbind.end(); ++i) { + } else if (gArgs.IsArgSet("-rpcbind")) { // Specific bind address + for (const std::string& strRPCBind : gArgs.GetArgs("-rpcbind")) { int port = defaultPort; std::string host; - SplitHostPort(*i, port, host); + SplitHostPort(strRPCBind, port, host); endpoints.push_back(std::make_pair(host, port)); } } else { // No specific bind address specified, bind to any diff --git a/src/init.cpp b/src/init.cpp index 1e7e388a52..33023a1800 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -25,6 +25,8 @@ #include "netbase.h" #include "net.h" #include "net_processing.h" +#include "policy/feerate.h" +#include "policy/fees.h" #include "policy/policy.h" #include "rpc/server.h" #include "rpc/register.h" @@ -57,7 +59,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/bind.hpp> -#include <boost/function.hpp> #include <boost/interprocess/sync/file_lock.hpp> #include <boost/thread.hpp> #include <openssl/crypto.h> @@ -145,9 +146,9 @@ class CCoinsViewErrorCatcher : public CCoinsViewBacked { public: CCoinsViewErrorCatcher(CCoinsView* view) : CCoinsViewBacked(view) {} - bool GetCoins(const uint256 &txid, CCoins &coins) const { + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override { try { - return CCoinsViewBacked::GetCoins(txid, coins); + return CCoinsViewBacked::GetCoin(outpoint, coin); } catch(const std::runtime_error& e) { uiInterface.ThreadSafeMessageBox(_("Error reading from database, shutting down."), "", CClientUIInterface::MSG_ERROR); LogPrintf("Error reading from database: %s\n", e.what()); @@ -207,15 +208,17 @@ void Shutdown() StopTorControl(); UnregisterNodeSignals(GetNodeSignals()); - if (fDumpMempoolLater) + if (fDumpMempoolLater && GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); + } if (fFeeEstimatesInitialized) { + ::feeEstimator.FlushUnconfirmed(::mempool); fs::path est_path = GetDataDir() / FEE_ESTIMATES_FILENAME; CAutoFile est_fileout(fsbridge::fopen(est_path, "wb"), SER_DISK, CLIENT_VERSION); if (!est_fileout.IsNull()) - mempool.WriteFeeEstimates(est_fileout); + ::feeEstimator.Write(est_fileout); else LogPrintf("%s: Failed to write fee estimates to %s\n", __func__, est_path.string()); fFeeEstimatesInitialized = false; @@ -326,6 +329,10 @@ void OnRPCPreCommand(const CRPCCommand& cmd) std::string HelpMessage(HelpMessageMode mode) { + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); + const auto defaultChainParams = CreateChainParams(CBaseChainParams::MAIN); + const auto testnetChainParams = CreateChainParams(CBaseChainParams::TESTNET); const bool showDebug = GetBoolArg("-help-debug", false); // When adding new options to the categories, please keep and ensure alphabetical ordering. @@ -337,7 +344,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-blocknotify=<cmd>", _("Execute command when the best block changes (%s in cmd is replaced by block hash)")); if (showDebug) strUsage += HelpMessageOpt("-blocksonly", strprintf(_("Whether to operate in a blocks only mode (default: %u)"), DEFAULT_BLOCKSONLY)); - strUsage +=HelpMessageOpt("-assumevalid=<hex>", strprintf(_("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)"), Params(CBaseChainParams::MAIN).GetConsensus().defaultAssumeValid.GetHex(), Params(CBaseChainParams::TESTNET).GetConsensus().defaultAssumeValid.GetHex())); + strUsage +=HelpMessageOpt("-assumevalid=<hex>", strprintf(_("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)"), defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex())); strUsage += HelpMessageOpt("-conf=<file>", strprintf(_("Specify configuration file (default: %s)"), BITCOIN_CONF_FILENAME)); if (mode == HMM_BITCOIND) { @@ -353,6 +360,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE)); strUsage += HelpMessageOpt("-mempoolexpiry=<n>", strprintf(_("Do not keep transactions in the mempool longer than <n> hours (default: %u)"), DEFAULT_MEMPOOL_EXPIRY)); + strUsage += HelpMessageOpt("-persistmempool", strprintf(_("Whether to save the mempool on shutdown and load on restart (default: %u)"), DEFAULT_PERSIST_MEMPOOL)); strUsage += HelpMessageOpt("-blockreconstructionextratxn=<n>", strprintf(_("Extra transactions to keep in memory for compact block reconstructions (default: %u)"), DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN)); strUsage += HelpMessageOpt("-par=<n>", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); @@ -390,7 +398,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-onlynet=<net>", _("Only connect to nodes in network <net> (ipv4, ipv6 or onion)")); strUsage += HelpMessageOpt("-permitbaremultisig", strprintf(_("Relay non-P2SH multisig (default: %u)"), DEFAULT_PERMIT_BAREMULTISIG)); strUsage += HelpMessageOpt("-peerbloomfilters", strprintf(_("Support filtering of blocks and transaction with bloom filters (default: %u)"), DEFAULT_PEERBLOOMFILTERS)); - strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), Params(CBaseChainParams::MAIN).GetDefaultPort(), Params(CBaseChainParams::TESTNET).GetDefaultPort())); + strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort())); strUsage += HelpMessageOpt("-proxy=<ip:port>", _("Connect through SOCKS5 proxy")); strUsage += HelpMessageOpt("-proxyrandomize", strprintf(_("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)"), DEFAULT_PROXYRANDOMIZE)); strUsage += HelpMessageOpt("-seednode=<ip>", _("Connect to a node to retrieve peer addresses, and disconnect")); @@ -427,14 +435,16 @@ std::string HelpMessage(HelpMessageMode mode) { strUsage += HelpMessageOpt("-checkblocks=<n>", strprintf(_("How many blocks to check at startup (default: %u, 0 = all)"), DEFAULT_CHECKBLOCKS)); strUsage += HelpMessageOpt("-checklevel=<n>", strprintf(_("How thorough the block verification of -checkblocks is (0-4, default: %u)"), DEFAULT_CHECKLEVEL)); - strUsage += HelpMessageOpt("-checkblockindex", strprintf("Do a full consistency check for mapBlockIndex, setBlockIndexCandidates, chainActive and mapBlocksUnlinked occasionally. Also sets -checkmempool (default: %u)", Params(CBaseChainParams::MAIN).DefaultConsistencyChecks())); - strUsage += HelpMessageOpt("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u)", Params(CBaseChainParams::MAIN).DefaultConsistencyChecks())); + strUsage += HelpMessageOpt("-checkblockindex", strprintf("Do a full consistency check for mapBlockIndex, setBlockIndexCandidates, chainActive and mapBlocksUnlinked occasionally. Also sets -checkmempool (default: %u)", defaultChainParams->DefaultConsistencyChecks())); + strUsage += HelpMessageOpt("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u)", defaultChainParams->DefaultConsistencyChecks())); strUsage += HelpMessageOpt("-checkpoints", strprintf("Disable expensive verification for known chain history (default: %u)", DEFAULT_CHECKPOINTS_ENABLED)); strUsage += HelpMessageOpt("-disablesafemode", strprintf("Disable safemode, override a real safe mode event (default: %u)", DEFAULT_DISABLE_SAFEMODE)); strUsage += HelpMessageOpt("-testsafemode", strprintf("Force safe mode (default: %u)", DEFAULT_TESTSAFEMODE)); strUsage += HelpMessageOpt("-dropmessagestest=<n>", "Randomly drop 1 of every <n> network messages"); strUsage += HelpMessageOpt("-fuzzmessagestest=<n>", "Randomly fuzz 1 of every <n> network messages"); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", DEFAULT_STOPAFTERBLOCKIMPORT)); + strUsage += HelpMessageOpt("-stopatheight", strprintf("Stop running after reaching the given height in the main chain (default: %u)", DEFAULT_STOPATHEIGHT)); + strUsage += HelpMessageOpt("-limitancestorcount=<n>", strprintf("Do not accept transactions if number of in-mempool ancestors is <n> or more (default: %u)", DEFAULT_ANCESTOR_LIMIT)); strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); @@ -467,7 +477,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageGroup(_("Node relay options:")); if (showDebug) { - strUsage += HelpMessageOpt("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", !Params(CBaseChainParams::TESTNET).RequireStandard())); + strUsage += HelpMessageOpt("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", defaultChainParams->RequireStandard())); strUsage += HelpMessageOpt("-incrementalrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to define cost of relay, used for mempool limiting and BIP 125 replacement. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_INCREMENTAL_RELAY_FEE))); strUsage += HelpMessageOpt("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to defined dust, the value of an output such that it will cost about 1/3 of its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE))); } @@ -495,7 +505,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-rpcuser=<user>", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcauth=<userpw>", _("Username and hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcuser. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times")); - strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Listen for JSON-RPC connections on <port> (default: %u or testnet: %u)"), BaseParams(CBaseChainParams::MAIN).RPCPort(), BaseParams(CBaseChainParams::TESTNET).RPCPort())); + strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Listen for JSON-RPC connections on <port> (default: %u or testnet: %u)"), defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort())); strUsage += HelpMessageOpt("-rpcallowip=<ip>", _("Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times")); strUsage += HelpMessageOpt("-rpcserialversion", strprintf(_("Sets the serialization of raw transaction or block hex returned in non-verbose mode, non-segwit(0) or segwit(1) (default: %d)"), DEFAULT_RPC_SERIALIZE_VERSION)); strUsage += HelpMessageOpt("-rpcthreads=<n>", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), DEFAULT_HTTP_THREADS)); @@ -676,8 +686,10 @@ void ThreadImport(std::vector<fs::path> vImportFiles) StartShutdown(); } } // End scope of CImportingNow - LoadMempool(); - fDumpMempoolLater = !fRequestShutdown; + if (GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + LoadMempool(); + fDumpMempoolLater = !fRequestShutdown; + } } /** Sanity checks @@ -734,7 +746,7 @@ void InitParameterInteraction() LogPrintf("%s: parameter interaction: -whitebind set -> setting -listen=1\n", __func__); } - if (mapMultiArgs.count("-connect") && mapMultiArgs.at("-connect").size() > 0) { + if (gArgs.IsArgSet("-connect")) { // when only connecting to trusted nodes, do not seed via DNS, or listen by default if (SoftSetBoolArg("-dnsseed", false)) LogPrintf("%s: parameter interaction: -connect set -> setting -dnsseed=0\n", __func__); @@ -888,8 +900,8 @@ bool AppInitParameterInteraction() // Make sure enough file descriptors are available int nBind = std::max( - (mapMultiArgs.count("-bind") ? mapMultiArgs.at("-bind").size() : 0) + - (mapMultiArgs.count("-whitebind") ? mapMultiArgs.at("-whitebind").size() : 0), size_t(1)); + (gArgs.IsArgSet("-bind") ? gArgs.GetArgs("-bind").size() : 0) + + (gArgs.IsArgSet("-whitebind") ? gArgs.GetArgs("-whitebind").size() : 0), size_t(1)); nUserMaxConnections = GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); nMaxConnections = std::max(nUserMaxConnections, 0); @@ -904,9 +916,9 @@ bool AppInitParameterInteraction() InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags - if (mapMultiArgs.count("-debug") > 0) { + if (gArgs.IsArgSet("-debug")) { // Special-case: if -debug=0/-nodebug is set, turn off debugging messages - const std::vector<std::string>& categories = mapMultiArgs.at("-debug"); + const std::vector<std::string> categories = gArgs.GetArgs("-debug"); if (find(categories.begin(), categories.end(), std::string("0")) == categories.end()) { for (const auto& cat : categories) { @@ -921,9 +933,8 @@ bool AppInitParameterInteraction() } // Now remove the logging categories which were explicitly excluded - if (mapMultiArgs.count("-debugexclude") > 0) { - const std::vector<std::string>& excludedCategories = mapMultiArgs.at("-debugexclude"); - for (const auto& cat : excludedCategories) { + if (gArgs.IsArgSet("-debugexclude")) { + for (const std::string& cat : gArgs.GetArgs("-debugexclude")) { uint32_t flag = 0; if (!GetLogCategory(&flag, &cat)) { InitWarning(strprintf(_("Unsupported logging category %s=%s."), "-debugexclude", cat)); @@ -1093,15 +1104,14 @@ bool AppInitParameterInteraction() fEnableReplacement = (std::find(vstrReplacementModes.begin(), vstrReplacementModes.end(), "fee") != vstrReplacementModes.end()); } - if (mapMultiArgs.count("-bip9params")) { + if (gArgs.IsArgSet("-bip9params")) { // Allow overriding BIP9 parameters for testing if (!chainparams.MineBlocksOnDemand()) { return InitError("BIP9 parameters may only be overridden on regtest."); } - const std::vector<std::string>& deployments = mapMultiArgs.at("-bip9params"); - for (auto i : deployments) { + for (const std::string& strDeployment : gArgs.GetArgs("-bip9params")) { std::vector<std::string> vDeploymentParams; - boost::split(vDeploymentParams, i, boost::is_any_of(":")); + boost::split(vDeploymentParams, strDeployment, boost::is_any_of(":")); if (vDeploymentParams.size() != 3) { return InitError("BIP9 parameters malformed, expecting deployment:start:end"); } @@ -1116,7 +1126,7 @@ bool AppInitParameterInteraction() for (int j=0; j<(int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) { if (vDeploymentParams[0].compare(VersionBitsDeploymentInfo[j].name) == 0) { - UpdateRegtestBIP9Parameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); + UpdateBIP9Parameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); found = true; LogPrintf("Setting BIP9 activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout); break; @@ -1247,8 +1257,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; - if (mapMultiArgs.count("-uacomment")) { - BOOST_FOREACH(std::string cmt, mapMultiArgs.at("-uacomment")) + if (gArgs.IsArgSet("-uacomment")) { + BOOST_FOREACH(std::string cmt, gArgs.GetArgs("-uacomment")) { if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) return InitError(strprintf(_("User Agent comment (%s) contains unsafe characters."), cmt)); @@ -1261,9 +1271,9 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) strSubVersion.size(), MAX_SUBVERSION_LENGTH)); } - if (mapMultiArgs.count("-onlynet")) { + if (gArgs.IsArgSet("-onlynet")) { std::set<enum Network> nets; - BOOST_FOREACH(const std::string& snet, mapMultiArgs.at("-onlynet")) { + BOOST_FOREACH(const std::string& snet, gArgs.GetArgs("-onlynet")) { enum Network net = ParseNetwork(snet); if (net == NET_UNROUTABLE) return InitError(strprintf(_("Unknown network specified in -onlynet: '%s'"), snet)); @@ -1276,8 +1286,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (mapMultiArgs.count("-whitelist")) { - BOOST_FOREACH(const std::string& net, mapMultiArgs.at("-whitelist")) { + if (gArgs.IsArgSet("-whitelist")) { + BOOST_FOREACH(const std::string& net, gArgs.GetArgs("-whitelist")) { CSubNet subnet; LookupSubNet(net.c_str(), subnet); if (!subnet.IsValid()) @@ -1338,16 +1348,16 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) if (fListen) { bool fBound = false; - if (mapMultiArgs.count("-bind")) { - BOOST_FOREACH(const std::string& strBind, mapMultiArgs.at("-bind")) { + if (gArgs.IsArgSet("-bind")) { + BOOST_FOREACH(const std::string& strBind, gArgs.GetArgs("-bind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, GetListenPort(), false)) return InitError(ResolveErrMsg("bind", strBind)); fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR)); } } - if (mapMultiArgs.count("-whitebind")) { - BOOST_FOREACH(const std::string& strBind, mapMultiArgs.at("-whitebind")) { + if (gArgs.IsArgSet("-whitebind")) { + BOOST_FOREACH(const std::string& strBind, gArgs.GetArgs("-whitebind")) { CService addrBind; if (!Lookup(strBind.c_str(), addrBind, 0, false)) return InitError(ResolveErrMsg("whitebind", strBind)); @@ -1356,7 +1366,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) fBound |= Bind(connman, addrBind, (BF_EXPLICIT | BF_REPORT_ERROR | BF_WHITELIST)); } } - if (!mapMultiArgs.count("-bind") && !mapMultiArgs.count("-whitebind")) { + if (!gArgs.IsArgSet("-bind") && !gArgs.IsArgSet("-whitebind")) { struct in_addr inaddr_any; inaddr_any.s_addr = INADDR_ANY; fBound |= Bind(connman, CService(in6addr_any, GetListenPort()), BF_NONE); @@ -1366,8 +1376,8 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) return InitError(_("Failed to listen on any port. Use -listen=0 if you want this.")); } - if (mapMultiArgs.count("-externalip")) { - BOOST_FOREACH(const std::string& strAddr, mapMultiArgs.at("-externalip")) { + if (gArgs.IsArgSet("-externalip")) { + BOOST_FOREACH(const std::string& strAddr, gArgs.GetArgs("-externalip")) { CService addrLocal; if (Lookup(strAddr.c_str(), addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) AddLocal(addrLocal, LOCAL_MANUAL); @@ -1376,11 +1386,6 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) } } - if (mapMultiArgs.count("-seednode")) { - BOOST_FOREACH(const std::string& strDest, mapMultiArgs.at("-seednode")) - connman.AddOneShot(strDest); - } - #if ENABLE_ZMQ pzmqNotificationInterface = CZMQNotificationInterface::Create(); @@ -1445,6 +1450,12 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (fPruneMode) CleanupBlockRevFiles(); + } else { + // If necessary, upgrade from older database format. + if (!pcoinsdbview->Upgrade()) { + strLoadError = _("Error upgrading chainstate database"); + break; + } } if (!LoadBlockIndex(chainparams)) { @@ -1550,7 +1561,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) CAutoFile est_filein(fsbridge::fopen(est_path, "rb"), SER_DISK, CLIENT_VERSION); // Allowed to fail as this file IS missing on first startup. if (!est_filein.IsNull()) - mempool.ReadFeeEstimates(est_filein); + ::feeEstimator.Read(est_filein); fFeeEstimatesInitialized = true; // ********************************************************* Step 8: load wallet @@ -1603,9 +1614,9 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) uiInterface.NotifyBlockTip.connect(BlockNotifyCallback); std::vector<fs::path> vImportFiles; - if (mapMultiArgs.count("-loadblock")) + if (gArgs.IsArgSet("-loadblock")) { - BOOST_FOREACH(const std::string& strFile, mapMultiArgs.at("-loadblock")) + BOOST_FOREACH(const std::string& strFile, gArgs.GetArgs("-loadblock")) vImportFiles.push_back(strFile); } @@ -1649,6 +1660,10 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) connOptions.nMaxOutboundTimeframe = nMaxOutboundTimeframe; connOptions.nMaxOutboundLimit = nMaxOutboundLimit; + if (gArgs.IsArgSet("-seednode")) { + connOptions.vSeedNodes = gArgs.GetArgs("-seednode"); + } + if (!connman.Start(scheduler, strNodeError, connOptions)) return InitError(strNodeError); diff --git a/src/key.cpp b/src/key.cpp index b4f0dc8202..5a75647f1a 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -131,14 +131,6 @@ void CKey::MakeNewKey(bool fCompressedIn) { fCompressed = fCompressedIn; } -bool CKey::SetPrivKey(const CPrivKey &privkey, bool fCompressedIn) { - if (!ec_privkey_import_der(secp256k1_context_sign, (unsigned char*)begin(), &privkey[0], privkey.size())) - return false; - fCompressed = fCompressedIn; - fValid = true; - return true; -} - CPrivKey CKey::GetPrivKey() const { assert(fValid); CPrivKey privkey; @@ -94,9 +94,6 @@ public: //! Check whether the public key corresponding to this private key is (to be) compressed. bool IsCompressed() const { return fCompressed; } - //! Initialize from a CPrivKey (serialized OpenSSL private key data). - bool SetPrivKey(const CPrivKey& vchPrivKey, bool fCompressed); - //! Generate a new private key using a cryptographic PRNG. void MakeNewKey(bool fCompressed); diff --git a/src/memusage.h b/src/memusage.h index 81e8702954..b69acafffd 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -12,6 +12,8 @@ #include <map> #include <set> #include <vector> +#include <unordered_map> +#include <unordered_set> #include <boost/foreach.hpp> #include <boost/unordered_set.hpp> @@ -149,7 +151,7 @@ static inline size_t DynamicUsage(const std::shared_ptr<X>& p) // Boost data structures template<typename X> -struct boost_unordered_node : private X +struct unordered_node : private X { private: void* ptr; @@ -158,13 +160,25 @@ private: template<typename X, typename Y> static inline size_t DynamicUsage(const boost::unordered_set<X, Y>& s) { - return MallocUsage(sizeof(boost_unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); + return MallocUsage(sizeof(unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); } template<typename X, typename Y, typename Z> static inline size_t DynamicUsage(const boost::unordered_map<X, Y, Z>& m) { - return MallocUsage(sizeof(boost_unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); + return MallocUsage(sizeof(unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); +} + +template<typename X, typename Y> +static inline size_t DynamicUsage(const std::unordered_set<X, Y>& s) +{ + return MallocUsage(sizeof(unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); +} + +template<typename X, typename Y, typename Z> +static inline size_t DynamicUsage(const std::unordered_map<X, Y, Z>& m) +{ + return MallocUsage(sizeof(unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); } } diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index e3f3e4621a..78d7cd6001 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -65,7 +65,7 @@ uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::ve } else { // calculate left hash uint256 left = CalcHash(height-1, pos*2, vTxid), right; - // calculate right hash if not beyond the end of the array - copy left hash otherwise1 + // calculate right hash if not beyond the end of the array - copy left hash otherwise if (pos*2+1 < CalcTreeWidth(height-1)) right = CalcHash(height-1, pos*2+1, vTxid); else diff --git a/src/merkleblock.h b/src/merkleblock.h index 73cbf670ee..de4c5c8d29 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -23,7 +23,7 @@ * storing a bit for each traversed node, signifying whether the node is the * parent of at least one matched leaf txid (or a matched txid itself). In * case we are at the leaf level, or this bit is 0, its merkle node hash is - * stored, and its children are not explorer further. Otherwise, no hash is + * stored, and its children are not explored further. Otherwise, no hash is * stored, but we recurse into both (or the only) child branch. During * decoding, the same depth-first traversal is performed, consuming bits and * hashes as they written during encoding. diff --git a/src/miner.cpp b/src/miner.cpp index 386d75c4be..28b6f23d56 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -10,11 +10,13 @@ #include "chainparams.h" #include "coins.h" #include "consensus/consensus.h" +#include "consensus/tx_verify.h" #include "consensus/merkle.h" #include "consensus/validation.h" #include "hash.h" #include "validation.h" #include "net.h" +#include "policy/feerate.h" #include "policy/policy.h" #include "pow.h" #include "primitives/transaction.h" @@ -317,9 +319,7 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already bool BlockAssembler::SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx) { assert (it != mempool.mapTx.end()); - if (mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it)) - return true; - return false; + return mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it); } void BlockAssembler::SortForBlock(const CTxMemPool::setEntries& package, CTxMemPool::txiter entry, std::vector<CTxMemPool::txiter>& sortedEntries) diff --git a/src/net.cpp b/src/net.cpp index cf94faf854..14ac5618eb 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -148,7 +148,7 @@ static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6> &vSeedsIn // one by discovery. CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices) { - CAddress ret(CService(CNetAddr(),GetListenPort()), NODE_NONE); + CAddress ret(CService(CNetAddr(),GetListenPort()), nLocalServices); CService addr; if (GetLocal(addr, paddrPeer)) { @@ -340,6 +340,22 @@ bool CConnman::CheckIncomingNonce(uint64_t nonce) return true; } +/** Get the bind address for a socket as CAddress */ +static CAddress GetBindAddress(SOCKET sock) +{ + CAddress addr_bind; + struct sockaddr_storage sockaddr_bind; + socklen_t sockaddr_bind_len = sizeof(sockaddr_bind); + if (sock != INVALID_SOCKET) { + if (!getsockname(sock, (struct sockaddr*)&sockaddr_bind, &sockaddr_bind_len)) { + addr_bind.SetSockAddr((const struct sockaddr*)&sockaddr_bind); + } else { + LogPrint(BCLog::NET, "Warning: getsockname failed\n"); + } + } + return addr_bind; +} + CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCountFailure) { if (pszDest == NULL) { @@ -393,7 +409,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo // Add node NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); - CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, pszDest ? pszDest : "", false); + CAddress addr_bind = GetBindAddress(hSocket); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addrConnect, CalculateKeyedNetGroup(addrConnect), nonce, addr_bind, pszDest ? pszDest : "", false); pnode->nServicesExpected = ServiceFlags(addrConnect.nServices & nRelevantServices); pnode->AddRef(); @@ -418,10 +435,10 @@ void CConnman::DumpBanlist() CBanDB bandb; banmap_t banmap; - SetBannedSetDirty(false); GetBanned(banmap); - if (!bandb.Write(banmap)) - SetBannedSetDirty(true); + if (bandb.Write(banmap)) { + SetBannedSetDirty(false); + } LogPrint(BCLog::NET, "Flushed %d banned node ips/subnets to banlist.dat %dms\n", banmap.size(), GetTimeMillis() - nStart); @@ -541,6 +558,8 @@ bool CConnman::Unban(const CSubNet &subNet) { void CConnman::GetBanned(banmap_t &banMap) { LOCK(cs_setBanned); + // Sweep the banlist so expired bans are not returned + SweepBanned(); banMap = setBanned; //create a thread safe copy } @@ -633,6 +652,7 @@ void CNode::copyStats(CNodeStats &stats) stats.nodeid = this->GetId(); X(nServices); X(addr); + X(addrBind); { LOCK(cs_filter); X(fRelayTxes); @@ -708,7 +728,7 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete handled = msg.readData(pch, nBytes); if (handled < 0) - return false; + return false; if (msg.in_data && msg.hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { LogPrint(BCLog::NET, "Oversized message from peer=%i, disconnecting\n", GetId()); @@ -786,7 +806,7 @@ int CNetMessage::readHeader(const char *pch, unsigned int nBytes) // reject messages larger than MAX_SIZE if (hdr.nMessageSize > MAX_SIZE) - return -1; + return -1; // switch state to reading message data in_data = true; @@ -946,7 +966,7 @@ bool CConnman::AttemptToEvictConnection() continue; if (node->fDisconnect) continue; - NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, + NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, node->nLastBlockTime, node->nLastTXTime, (node->nServices & nRelevantServices) == nRelevantServices, node->fRelayTxes, node->pfilter != NULL, node->addr, node->nKeyedNetGroup}; @@ -1034,9 +1054,11 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { int nInbound = 0; int nMaxInbound = nMaxConnections - (nMaxOutbound + nMaxFeeler); - if (hSocket != INVALID_SOCKET) - if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) + if (hSocket != INVALID_SOCKET) { + if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) { LogPrintf("Warning: Unknown socket family\n"); + } + } bool whitelisted = hListenSocket.whitelisted || IsWhitelistedRange(addr); { @@ -1069,12 +1091,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { // According to the internet TCP_NODELAY is not carried into accepted sockets // on all platforms. Set it again here just to be sure. - int set = 1; -#ifdef WIN32 - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); -#else - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&set, sizeof(int)); -#endif + SetSocketNoDelay(hSocket); if (IsBanned(addr) && !whitelisted) { @@ -1095,8 +1112,9 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { NodeId id = GetNewNodeId(); uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + CAddress addr_bind = GetBindAddress(hSocket); - CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, "", true); + CNode* pnode = new CNode(id, nLocalServices, GetBestHeight(), hSocket, addr, CalculateKeyedNetGroup(addr), nonce, addr_bind, "", true); pnode->AddRef(); pnode->fWhitelisted = whitelisted; GetNodeSignals().InitializeNode(pnode, *this); @@ -1299,59 +1317,55 @@ void CConnman::ThreadSocketHandler() } if (recvSet || errorSet) { + // typical socket buffer is 8K-64K + char pchBuf[0x10000]; + int nBytes = 0; { - { - // typical socket buffer is 8K-64K - char pchBuf[0x10000]; - int nBytes = 0; - { - LOCK(pnode->cs_hSocket); - if (pnode->hSocket == INVALID_SOCKET) - continue; - nBytes = recv(pnode->hSocket, pchBuf, sizeof(pchBuf), MSG_DONTWAIT); - } - if (nBytes > 0) - { - bool notify = false; - if (!pnode->ReceiveMsgBytes(pchBuf, nBytes, notify)) - pnode->CloseSocketDisconnect(); - RecordBytesRecv(nBytes); - if (notify) { - size_t nSizeAdded = 0; - auto it(pnode->vRecvMsg.begin()); - for (; it != pnode->vRecvMsg.end(); ++it) { - if (!it->complete()) - break; - nSizeAdded += it->vRecv.size() + CMessageHeader::HEADER_SIZE; - } - { - LOCK(pnode->cs_vProcessMsg); - pnode->vProcessMsg.splice(pnode->vProcessMsg.end(), pnode->vRecvMsg, pnode->vRecvMsg.begin(), it); - pnode->nProcessQueueSize += nSizeAdded; - pnode->fPauseRecv = pnode->nProcessQueueSize > nReceiveFloodSize; - } - WakeMessageHandler(); - } - } - else if (nBytes == 0) - { - // socket closed gracefully - if (!pnode->fDisconnect) { - LogPrint(BCLog::NET, "socket closed\n"); - } - pnode->CloseSocketDisconnect(); + LOCK(pnode->cs_hSocket); + if (pnode->hSocket == INVALID_SOCKET) + continue; + nBytes = recv(pnode->hSocket, pchBuf, sizeof(pchBuf), MSG_DONTWAIT); + } + if (nBytes > 0) + { + bool notify = false; + if (!pnode->ReceiveMsgBytes(pchBuf, nBytes, notify)) + pnode->CloseSocketDisconnect(); + RecordBytesRecv(nBytes); + if (notify) { + size_t nSizeAdded = 0; + auto it(pnode->vRecvMsg.begin()); + for (; it != pnode->vRecvMsg.end(); ++it) { + if (!it->complete()) + break; + nSizeAdded += it->vRecv.size() + CMessageHeader::HEADER_SIZE; } - else if (nBytes < 0) { - // error - int nErr = WSAGetLastError(); - if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) - { - if (!pnode->fDisconnect) - LogPrintf("socket recv error %s\n", NetworkErrorString(nErr)); - pnode->CloseSocketDisconnect(); - } + LOCK(pnode->cs_vProcessMsg); + pnode->vProcessMsg.splice(pnode->vProcessMsg.end(), pnode->vRecvMsg, pnode->vRecvMsg.begin(), it); + pnode->nProcessQueueSize += nSizeAdded; + pnode->fPauseRecv = pnode->nProcessQueueSize > nReceiveFloodSize; } + WakeMessageHandler(); + } + } + else if (nBytes == 0) + { + // socket closed gracefully + if (!pnode->fDisconnect) { + LogPrint(BCLog::NET, "socket closed\n"); + } + pnode->CloseSocketDisconnect(); + } + else if (nBytes < 0) + { + // error + int nErr = WSAGetLastError(); + if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) + { + if (!pnode->fDisconnect) + LogPrintf("socket recv error %s\n", NetworkErrorString(nErr)); + pnode->CloseSocketDisconnect(); } } } @@ -1376,7 +1390,7 @@ void CConnman::ThreadSocketHandler() { if (pnode->nLastRecv == 0 || pnode->nLastSend == 0) { - LogPrint(BCLog::NET, "socket no message in first 60 seconds, %d %d from %d\n", pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->id); + LogPrint(BCLog::NET, "socket no message in first 60 seconds, %d %d from %d\n", pnode->nLastRecv != 0, pnode->nLastSend != 0, pnode->GetId()); pnode->fDisconnect = true; } else if (nTime - pnode->nLastSend > TIMEOUT_INTERVAL) @@ -1396,7 +1410,7 @@ void CConnman::ThreadSocketHandler() } else if (!pnode->fSuccessfullyConnected) { - LogPrintf("version handshake timeout from %d\n", pnode->id); + LogPrintf("version handshake timeout from %d\n", pnode->GetId()); pnode->fDisconnect = true; } } @@ -1585,6 +1599,9 @@ void CConnman::ThreadDNSAddressSeed() LogPrintf("Loading addresses from DNS seeds (could take a while)\n"); BOOST_FOREACH(const CDNSSeedData &seed, vSeeds) { + if (interruptNet) { + return; + } if (HaveNameProxy()) { AddOneShot(seed.host); } else { @@ -1602,6 +1619,9 @@ void CConnman::ThreadDNSAddressSeed() found++; } } + if (interruptNet) { + return; + } // TODO: The seed name resolve may fail, yielding an IP of [::], which results in // addrman assigning the same source to results from different seeds. // This should switch to a hard-coded stable dummy IP for each seed name, so that the @@ -1666,12 +1686,12 @@ void CConnman::ProcessOneShot() void CConnman::ThreadOpenConnections() { // Connect to specific addresses - if (mapMultiArgs.count("-connect") && mapMultiArgs.at("-connect").size() > 0) + if (gArgs.IsArgSet("-connect") && gArgs.GetArgs("-connect").size() > 0) { for (int64_t nLoop = 0;; nLoop++) { ProcessOneShot(); - BOOST_FOREACH(const std::string& strAddr, mapMultiArgs.at("-connect")) + BOOST_FOREACH(const std::string& strAddr, gArgs.GetArgs("-connect")) { CAddress addr(CService(), NODE_NONE); OpenNetworkConnection(addr, false, NULL, strAddr.c_str()); @@ -1722,11 +1742,17 @@ void CConnman::ThreadOpenConnections() // Only connect out to one peer per network group (/16 for IPv4). // Do this here so we don't have to critsect vNodes inside mapAddresses critsect. int nOutbound = 0; + int nOutboundRelevant = 0; std::set<std::vector<unsigned char> > setConnected; { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { if (!pnode->fInbound && !pnode->fAddnode) { + + // Count the peers that have all relevant services + if (pnode->fSuccessfullyConnected && !pnode->fFeeler && ((pnode->nServices & nRelevantServices) == nRelevantServices)) { + nOutboundRelevant++; + } // Netgroups for inbound and addnode peers are not excluded because our goal here // is to not use multiple of our limited outbound slots on a single netgroup // but inbound and addnode peers do not use our outbound slots. Inbound peers @@ -1744,9 +1770,9 @@ void CConnman::ThreadOpenConnections() // * Increase the number of connectable addresses in the tried table. // // Method: - // * Choose a random address from new and attempt to connect to it if we can connect + // * Choose a random address from new and attempt to connect to it if we can connect // successfully it is added to tried. - // * Start attempting feeler connections only after node finishes making outbound + // * Start attempting feeler connections only after node finishes making outbound // connections. // * Only make a feeler connection once every few minutes. // @@ -1790,14 +1816,27 @@ void CConnman::ThreadOpenConnections() continue; // only consider nodes missing relevant services after 40 failed attempts and only if less than half the outbound are up. - if ((addr.nServices & nRelevantServices) != nRelevantServices && (nTries < 40 || nOutbound >= (nMaxOutbound >> 1))) + ServiceFlags nRequiredServices = nRelevantServices; + if (nTries >= 40 && nOutbound < (nMaxOutbound >> 1)) { + nRequiredServices = REQUIRED_SERVICES; + } + + if ((addr.nServices & nRequiredServices) != nRequiredServices) { continue; + } // do not allow non-default ports, unless after 50 invalid addresses selected already if (addr.GetPort() != Params().GetDefaultPort() && nTries < 50) continue; addrConnect = addr; + + // regardless of the services assumed to be available, only require the minimum if half or more outbound have relevant services + if (nOutboundRelevant >= (nMaxOutbound >> 1)) { + addrConnect.nServices = REQUIRED_SERVICES; + } else { + addrConnect.nServices = nRequiredServices; + } break; } @@ -1873,8 +1912,8 @@ void CConnman::ThreadOpenAddedConnections() { { LOCK(cs_vAddedNodes); - if (mapMultiArgs.count("-addnode")) - vAddedNodes = mapMultiArgs.at("-addnode"); + if (gArgs.IsArgSet("-addnode")) + vAddedNodes = gArgs.GetArgs("-addnode"); } while (true) @@ -2213,6 +2252,10 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c SetBestHeight(connOptions.nBestHeight); + for (const auto& strDest : connOptions.vSeedNodes) { + AddOneShot(strDest); + } + clientInterface = connOptions.uiInterface; if (clientInterface) { clientInterface->InitMessage(_("Loading P2P addresses...")); @@ -2285,7 +2328,7 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c threadOpenAddedConnections = std::thread(&TraceThread<std::function<void()> >, "addcon", std::function<void()>(std::bind(&CConnman::ThreadOpenAddedConnections, this))); // Initiate outbound connections unless connect=0 - if (!mapMultiArgs.count("-connect") || mapMultiArgs.at("-connect").size() != 1 || mapMultiArgs.at("-connect")[0] != "0") + if (!gArgs.IsArgSet("-connect") || gArgs.GetArgs("-connect").size() != 1 || gArgs.GetArgs("-connect")[0] != "0") threadOpenConnections = std::thread(&TraceThread<std::function<void()> >, "opencon", std::function<void()>(std::bind(&CConnman::ThreadOpenConnections, this))); // Process messages @@ -2483,7 +2526,7 @@ bool CConnman::DisconnectNode(NodeId id) { LOCK(cs_vNodes); for(CNode* pnode : vNodes) { - if (id == pnode->id) { + if (id == pnode->GetId()) { pnode->fDisconnect = true; return true; } @@ -2617,14 +2660,15 @@ int CConnman::GetBestHeight() const unsigned int CConnman::GetReceiveFloodSize() const { return nReceiveFloodSize; } unsigned int CConnman::GetSendBufferSize() const{ return nSendBufferMaxSize; } -CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const std::string& addrNameIn, bool fInboundIn) : +CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string& addrNameIn, bool fInboundIn) : nTimeConnected(GetSystemTimeInSeconds()), addr(addrIn), + addrBind(addrBindIn), fInbound(fInboundIn), - id(idIn), nKeyedNetGroup(nKeyedNetGroupIn), addrKnown(5000, 0.001), filterInventoryKnown(50000, 0.000001), + id(idIn), nLocalHostNonce(nLocalHostNonceIn), nLocalServices(nLocalServicesIn), nMyStartingHeight(nMyStartingHeightIn), @@ -2740,7 +2784,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); size_t nTotalSize = nMessageSize + CMessageHeader::HEADER_SIZE; - LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->id); + LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.command.c_str()), nMessageSize, pnode->GetId()); std::vector<unsigned char> serializedHeader; serializedHeader.reserve(CMessageHeader::HEADER_SIZE); @@ -2778,7 +2822,7 @@ bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) CNode* found = nullptr; LOCK(cs_vNodes); for (auto&& pnode : vNodes) { - if(pnode->id == id) { + if(pnode->GetId() == id) { found = pnode; break; } @@ -11,10 +11,10 @@ #include "amount.h" #include "bloom.h" #include "compat.h" -#include "fs.h" #include "hash.h" #include "limitedmap.h" #include "netaddress.h" +#include "policy/feerate.h" #include "protocol.h" #include "random.h" #include "streams.h" @@ -36,7 +36,6 @@ #include <boost/foreach.hpp> #include <boost/signals2/signal.hpp> -class CAddrMan; class CScheduler; class CNode; @@ -92,7 +91,7 @@ static const ServiceFlags REQUIRED_SERVICES = NODE_NETWORK; // NOTE: When adjusting this, update rpcnet:setban's help ("24h") static const unsigned int DEFAULT_MISBEHAVING_BANTIME = 60 * 60 * 24; // Default 24-hour ban -typedef int NodeId; +typedef int64_t NodeId; struct AddedNodeInfo { @@ -145,6 +144,7 @@ public: unsigned int nReceiveFloodSize = 0; uint64_t nMaxOutboundTimeframe = 0; uint64_t nMaxOutboundLimit = 0; + std::vector<std::string> vSeedNodes; }; CConnman(uint64_t seed0, uint64_t seed1); ~CConnman(); @@ -234,8 +234,6 @@ public: void GetBanned(banmap_t &banmap); void SetBanned(const banmap_t &banmap); - void AddOneShot(const std::string& strDest); - bool AddNode(const std::string& node); bool RemoveAddedNode(const std::string& node); std::vector<AddedNodeInfo> GetAddedNodeInfo(); @@ -293,6 +291,7 @@ private: }; void ThreadOpenAddedConnections(); + void AddOneShot(const std::string& strDest); void ProcessOneShot(); void ThreadOpenConnections(); void ThreadMessageHandler(); @@ -505,8 +504,12 @@ public: double dPingTime; double dPingWait; double dMinPing; + // Our address, as reported by the peer std::string addrLocal; + // Address of this peer CAddress addr; + // Bind address of our side of the connection + CAddress addrBind; }; @@ -587,7 +590,10 @@ public: std::atomic<int64_t> nLastRecv; const int64_t nTimeConnected; std::atomic<int64_t> nTimeOffset; + // Address of this peer const CAddress addr; + // Bind address of our side of the connection + const CAddress addrBind; std::atomic<int> nVersion; // strSubVer is whatever byte array we read from the wire. However, this field is intended // to be printed out, displayed to humans in various forms and so on. So we sanitize it and @@ -613,7 +619,6 @@ public: CCriticalSection cs_filter; CBloomFilter* pfilter; std::atomic<int> nRefCount; - const NodeId id; const uint64_t nKeyedNetGroup; std::atomic_bool fPauseRecv; @@ -678,12 +683,13 @@ public: CAmount lastSentFeeFilter; int64_t nextSendTimeFeeFilter; - CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const std::string &addrNameIn = "", bool fInboundIn = false); + CNode(NodeId id, ServiceFlags nLocalServicesIn, int nMyStartingHeightIn, SOCKET hSocketIn, const CAddress &addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress &addrBindIn, const std::string &addrNameIn = "", bool fInboundIn = false); ~CNode(); private: CNode(const CNode&); void operator=(const CNode&); + const NodeId id; const uint64_t nLocalHostNonce; @@ -696,23 +702,24 @@ private: mutable CCriticalSection cs_addrName; std::string addrName; + // Our address, as reported by the peer CService addrLocal; mutable CCriticalSection cs_addrLocal; public: NodeId GetId() const { - return id; + return id; } uint64_t GetLocalNonce() const { - return nLocalHostNonce; + return nLocalHostNonce; } int GetMyStartingHeight() const { - return nMyStartingHeight; + return nMyStartingHeight; } - int GetRefCount() + int GetRefCount() const { assert(nRefCount >= 0); return nRefCount; @@ -724,7 +731,7 @@ public: { nRecvVersion = nVersionIn; } - int GetRecvVersion() + int GetRecvVersion() const { return nRecvVersion; } @@ -760,7 +767,7 @@ public: // after addresses were pushed. if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { - vAddrToSend[insecure_rand.rand32() % vAddrToSend.size()] = _addr; + vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; } else { vAddrToSend.push_back(_addr); } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index d881b8ddda..971db3427c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -303,6 +303,7 @@ void FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { assert(nPreferredDownload == 0); assert(nPeersWithValidatedDownloads == 0); } + LogPrint(BCLog::NET, "Cleared nodestate for peer=%d\n", nodeid); } // Requires cs_main. @@ -333,7 +334,7 @@ bool MarkBlockAsReceived(const uint256& hash) { // Requires cs_main. // returns false, still setting pit, if the block was already in flight from the same peer // pit will only be valid as long as the same cs_main lock is being held -bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, const CBlockIndex* pindex = NULL, std::list<QueuedBlock>::iterator** pit = NULL) { +bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const CBlockIndex* pindex = NULL, std::list<QueuedBlock>::iterator** pit = NULL) { CNodeState *state = State(nodeid); assert(state != NULL); @@ -753,8 +754,8 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pb const CTransaction& tx = *ptx; // Which orphan pool entries must we evict? - for (size_t j = 0; j < tx.vin.size(); j++) { - auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + for (const auto& txin : tx.vin) { + auto itByPrev = mapOrphanTransactionsByPrev.find(txin.prevout); if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { const CTransaction& orphanTx = *(*mi)->second.tx; @@ -774,10 +775,12 @@ void PeerLogicValidation::BlockConnected(const std::shared_ptr<const CBlock>& pb } } +// All of the following cache a recent block, and are protected by cs_most_recent_block static CCriticalSection cs_most_recent_block; static std::shared_ptr<const CBlock> most_recent_block; static std::shared_ptr<const CBlockHeaderAndShortTxIDs> most_recent_compact_block; static uint256 most_recent_block_hash; +static bool fWitnessesPresentInMostRecentCompactBlock; void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) { std::shared_ptr<const CBlockHeaderAndShortTxIDs> pcmpctblock = std::make_shared<const CBlockHeaderAndShortTxIDs> (*pblock, true); @@ -798,6 +801,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: most_recent_block_hash = hashBlock; most_recent_block = pblock; most_recent_compact_block = pcmpctblock; + fWitnessesPresentInMostRecentCompactBlock = fWitnessEnabled; } connman->ForEachNode([this, &pcmpctblock, pindex, &msgMaker, fWitnessEnabled, &hashBlock](CNode* pnode) { @@ -812,7 +816,7 @@ void PeerLogicValidation::NewPoWValidBlock(const CBlockIndex *pindex, const std: !PeerHasHeader(&state, pindex) && PeerHasHeader(&state, pindex->pprev)) { LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", "PeerLogicValidation::NewPoWValidBlock", - hashBlock.ToString(), pnode->id); + hashBlock.ToString(), pnode->GetId()); connman->PushMessage(pnode, msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock)); state.pindexBestHeaderSent = pindex; } @@ -907,12 +911,11 @@ bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) recentRejects->reset(); } - // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude - // requesting or processing some txs which have already been included in a block return recentRejects->contains(inv.hash) || mempool.exists(inv.hash) || mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoinsInCache(inv.hash); + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 0)) || // Best effort: only try output 0 and 1 + pcoinsTip->HaveCoinInCache(COutPoint(inv.hash, 1)); } case MSG_BLOCK: case MSG_WITNESS_BLOCK: @@ -947,7 +950,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman& connma auto sortfunc = [&best, &hasher, nRelayNodes](CNode* pnode) { if (pnode->nVersion >= CADDR_TIME_VERSION) { - uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize(); + uint64_t hashKey = CSipHasher(hasher).Write(pnode->GetId()).Finalize(); for (unsigned int i = 0; i < nRelayNodes; i++) { if (hashKey > best[i].first) { std::copy(best.begin() + i, best.begin() + nRelayNodes - 1, best.begin() + i + 1); @@ -990,6 +993,15 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam { bool send = false; BlockMap::iterator mi = mapBlockIndex.find(inv.hash); + std::shared_ptr<const CBlock> a_recent_block; + std::shared_ptr<const CBlockHeaderAndShortTxIDs> a_recent_compact_block; + bool fWitnessesPresentInARecentCompactBlock; + { + LOCK(cs_most_recent_block); + a_recent_block = most_recent_block; + a_recent_compact_block = most_recent_compact_block; + fWitnessesPresentInARecentCompactBlock = fWitnessesPresentInMostRecentCompactBlock; + } if (mi != mapBlockIndex.end()) { if (mi->second->nChainTx && !mi->second->IsValid(BLOCK_VALID_SCRIPTS) && @@ -999,11 +1011,6 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // before ActivateBestChain but after AcceptBlock). // In this case, we need to run ActivateBestChain prior to checking the relay // conditions below. - std::shared_ptr<const CBlock> a_recent_block; - { - LOCK(cs_most_recent_block); - a_recent_block = most_recent_block; - } CValidationState dummy; ActivateBestChain(dummy, Params(), a_recent_block); } @@ -1037,14 +1044,20 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // it's available before trying to send. if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) { - // Send block from disk - CBlock block; - if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) - assert(!"cannot load block from disk"); + std::shared_ptr<const CBlock> pblock; + if (a_recent_block && a_recent_block->GetHash() == (*mi).second->GetBlockHash()) { + pblock = a_recent_block; + } else { + // Send block from disk + std::shared_ptr<CBlock> pblockRead = std::make_shared<CBlock>(); + if (!ReadBlockFromDisk(*pblockRead, (*mi).second, consensusParams)) + assert(!"cannot load block from disk"); + pblock = pblockRead; + } if (inv.type == MSG_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block)); + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_WITNESS_BLOCK) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, block)); + connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); else if (inv.type == MSG_FILTERED_BLOCK) { bool sendMerkleBlock = false; @@ -1053,7 +1066,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam LOCK(pfrom->cs_filter); if (pfrom->pfilter) { sendMerkleBlock = true; - merkleBlock = CMerkleBlock(block, *pfrom->pfilter); + merkleBlock = CMerkleBlock(*pblock, *pfrom->pfilter); } } if (sendMerkleBlock) { @@ -1066,7 +1079,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *block.vtx[pair.first])); + connman.PushMessage(pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); } // else // no response @@ -1080,10 +1093,15 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam bool fPeerWantsWitness = State(pfrom->GetId())->fWantsCmpctWitness; int nSendFlags = fPeerWantsWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; if (CanDirectFetch(consensusParams) && mi->second->nHeight >= chainActive.Height() - MAX_CMPCTBLOCK_DEPTH) { - CBlockHeaderAndShortTxIDs cmpctblock(block, fPeerWantsWitness); - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); - } else - connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, block)); + if ((fPeerWantsWitness || !fWitnessesPresentInARecentCompactBlock) && a_recent_compact_block && a_recent_compact_block->header.GetHash() == mi->second->GetBlockHash()) { + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + } else { + CBlockHeaderAndShortTxIDs cmpctblock(*pblock, fPeerWantsWitness); + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, cmpctblock)); + } + } else { + connman.PushMessage(pfrom, msgMaker.Make(nSendFlags, NetMsgType::BLOCK, *pblock)); + } } // Trigger the peer node to send a getblocks request for the next batch of inventory @@ -1144,7 +1162,7 @@ void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParam } } -uint32_t GetFetchFlags(CNode* pfrom, const CBlockIndex* pprev, const Consensus::Params& chainparams) { +uint32_t GetFetchFlags(CNode* pfrom) { uint32_t nFetchFlags = 0; if ((pfrom->GetLocalServices() & NODE_WITNESS) && State(pfrom->GetId())->fHaveWitness) { nFetchFlags |= MSG_WITNESS_FLAG; @@ -1158,7 +1176,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac if (req.indexes[i] >= block.vtx.size()) { LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); + LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->GetId()); return; } resp.txn[i] = block.vtx[req.indexes[i]]; @@ -1171,7 +1189,7 @@ inline void static SendBlockTransactions(const CBlock& block, const BlockTransac bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CConnman& connman, const std::atomic<bool>& interruptMsgProc) { - LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); + LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); if (IsArgSet("-dropmessagestest") && GetRand(GetArg("-dropmessagestest", 0)) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -1250,7 +1268,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (pfrom->nServicesExpected & ~nServices) { - LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, nServices, pfrom->nServicesExpected); + LogPrint(BCLog::NET, "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->GetId(), nServices, pfrom->nServicesExpected); connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, strprintf("Expected to offer services %08x", pfrom->nServicesExpected))); pfrom->fDisconnect = true; @@ -1260,7 +1278,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version - LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, nVersion); + LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->GetId(), nVersion); connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION))); pfrom->fDisconnect = true; @@ -1362,7 +1380,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LogPrintf("receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", cleanSubVer, pfrom->nVersion, - pfrom->nStartingHeight, addrMe.ToString(), pfrom->id, + pfrom->nStartingHeight, addrMe.ToString(), pfrom->GetId(), remoteAddr); int64_t nTimeOffset = nTime - GetTime(); @@ -1532,19 +1550,15 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); - uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); - - std::vector<CInv> vToFetch; + uint32_t nFetchFlags = GetFetchFlags(pfrom); - for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) + for (CInv &inv : vInv) { - CInv &inv = vInv[nInv]; - if (interruptMsgProc) return true; bool fAlreadyHave = AlreadyHave(inv); - LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); + LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->GetId()); if (inv.type == MSG_TX) { inv.type |= nFetchFlags; @@ -1559,14 +1573,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // we now only provide a getheaders response here. When we receive the headers, we will // then ask for the blocks we need. connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash)); - LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->GetId()); } } else { pfrom->AddInventoryKnown(inv); if (fBlocksOnly) { - LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); + LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->GetId()); } else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) { pfrom->AskFor(inv); } @@ -1575,9 +1589,6 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Track requests for our stuff GetMainSignals().Inventory(inv.hash); } - - if (!vToFetch.empty()) - connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vToFetch)); } @@ -1592,10 +1603,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr return error("message getdata size() = %u", vInv.size()); } - LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->id); + LogPrint(BCLog::NET, "received getdata (%u invsz) peer=%d\n", vInv.size(), pfrom->GetId()); if (vInv.size() > 0) { - LogPrint(BCLog::NET, "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->id); + LogPrint(BCLog::NET, "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->GetId()); } pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); @@ -1635,7 +1646,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (pindex) pindex = chainActive.Next(pindex); int nLimit = 500; - LogPrint(BCLog::NET, "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->id); + LogPrint(BCLog::NET, "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->GetId()); for (; pindex; pindex = chainActive.Next(pindex)) { if (pindex->GetBlockHash() == hashStop) @@ -1685,7 +1696,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { - LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); + LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->GetId()); return true; } @@ -1697,7 +1708,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // might maliciously send lots of getblocktxn requests to trigger // expensive disk reads, because it will require the peer to // actually receive all the data read from disk over the network. - LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->id, MAX_BLOCKTXN_DEPTH); + LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block > %i deep", pfrom->GetId(), MAX_BLOCKTXN_DEPTH); CInv inv; inv.type = State(pfrom->GetId())->fWantsCmpctWitness ? MSG_WITNESS_BLOCK : MSG_BLOCK; inv.hash = req.blockhash; @@ -1722,7 +1733,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); if (IsInitialBlockDownload() && !pfrom->fWhitelisted) { - LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->GetId()); return true; } @@ -1747,7 +1758,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // we must use CBlocks, as CBlockHeaders won't include the 0x00 nTx count at the end std::vector<CBlock> vHeaders; int nLimit = MAX_HEADERS_RESULTS; - LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->id); + LogPrint(BCLog::NET, "getheaders %d to %s from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), pfrom->GetId()); for (; pindex; pindex = chainActive.Next(pindex)) { vHeaders.push_back(pindex->GetBlockHeader()); @@ -1777,7 +1788,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off if (!fRelayTxes && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) { - LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "transaction sent in violation of protocol peer=%d\n", pfrom->GetId()); return true; } @@ -1810,7 +1821,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr pfrom->nLastTXTime = GetTime(); LogPrint(BCLog::MEMPOOL, "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", - pfrom->id, + pfrom->GetId(), tx.GetHash().ToString(), mempool.size(), mempool.DynamicMemoryUsage() / 1000); @@ -1885,7 +1896,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } } if (!fRejectedParents) { - uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); + uint32_t nFetchFlags = GetFetchFlags(pfrom); BOOST_FOREACH(const CTxIn& txin, tx.vin) { CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); pfrom->AddInventoryKnown(_inv); @@ -1930,10 +1941,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // case. int nDoS = 0; if (!state.IsInvalid(nDoS) || nDoS == 0) { - LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->id); + LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->GetId()); RelayTransaction(tx, connman); } else { - LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->id, FormatStateMessage(state)); + LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->GetId(), FormatStateMessage(state)); } } } @@ -1945,7 +1956,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (state.IsInvalid(nDoS)) { LogPrint(BCLog::MEMPOOLREJ, "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), - pfrom->id, + pfrom->GetId(), FormatStateMessage(state)); if (state.GetRejectCode() > 0 && state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), @@ -1982,7 +1993,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); } - LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->GetId()); return true; } } @@ -2022,7 +2033,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We requested this block for some reason, but our mempool will probably be useless // so we just grab the block via normal getdata std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return true; @@ -2046,7 +2057,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { std::list<QueuedBlock>::iterator* queuedBlockIt = NULL; - if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex, &queuedBlockIt)) { + if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); else { @@ -2061,12 +2072,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block\n", pfrom->id); + LogPrintf("Peer %d sent us invalid compact block\n", pfrom->GetId()); return true; } else if (status == READ_STATUS_FAILED) { // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } @@ -2109,7 +2120,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // We requested this block, but its far into the future, so our // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return true; } else { @@ -2166,7 +2177,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || it->second.first != pfrom->GetId()) { - LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->GetId()); return true; } @@ -2175,12 +2186,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (status == READ_STATUS_INVALID) { MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist Misbehaving(pfrom->GetId(), 100); - LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->GetId()); return true; } else if (status == READ_STATUS_FAILED) { // Might have collided, fall back to getdata now :( std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()), resp.blockhash)); + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received @@ -2263,7 +2274,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr headers[0].GetHash().ToString(), headers[0].hashPrevBlock.ToString(), pindexBestHeader->nHeight, - pfrom->id, nodestate->nUnconnectingHeaders); + pfrom->GetId(), nodestate->nUnconnectingHeaders); // Set hashLastUnknownBlock for this peer, so that if we // eventually get the headers - even from a different peer - // we can use this peer to download. @@ -2301,7 +2312,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); CNodeState *nodestate = State(pfrom->GetId()); if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->id, nodestate->nUnconnectingHeaders); + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->GetId(), nodestate->nUnconnectingHeaders); } nodestate->nUnconnectingHeaders = 0; @@ -2312,7 +2323,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. - LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); + LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->GetId(), pfrom->nStartingHeight); connman.PushMessage(pfrom, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256())); } @@ -2348,11 +2359,11 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Can't download any more from this peer break; } - uint32_t nFetchFlags = GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()); + uint32_t nFetchFlags = GetFetchFlags(pfrom); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex); + MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom->id); + pindex->GetBlockHash().ToString(), pfrom->GetId()); } if (vGetData.size() > 1) { LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", @@ -2375,7 +2386,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); vRecv >> *pblock; - LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->id); + LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom->GetId()); // Process all blocks from whitelisted peers, even if not requested, // unless we're still syncing with the network. @@ -2407,14 +2418,14 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr // Making nodes which are behind NAT and can only make outgoing connections ignore // the getaddr message mitigates the attack. if (!pfrom->fInbound) { - LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->GetId()); return true; } // Only send one GetAddr response per connection to reduce resource waste // and discourage addr stamping of INV announcements. if (pfrom->fSentAddr) { - LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->id); + LogPrint(BCLog::NET, "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->GetId()); return true; } pfrom->fSentAddr = true; @@ -2515,7 +2526,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (!(sProblem.empty())) { LogPrint(BCLog::NET, "pong peer=%d: %s, %x expected, %x received, %u bytes\n", - pfrom->id, + pfrom->GetId(), sProblem, pfrom->nPingNonceSent, nonce, @@ -2592,7 +2603,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(pfrom->cs_feeFilter); pfrom->minFeeFilter = newFeeFilter; } - LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->id); + LogPrint(BCLog::NET, "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->GetId()); } } @@ -2603,7 +2614,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr else { // Ignore unknown commands for extensibility - LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); + LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->GetId()); } @@ -2663,100 +2674,100 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic<bool>& i // this maintains the order of responses if (!pfrom->vRecvGetData.empty()) return true; - // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - return false; + // Don't bother if send buffer is too full to respond anyway + if (pfrom->fPauseSend) + return false; - std::list<CNetMessage> msgs; - { - LOCK(pfrom->cs_vProcessMsg); - if (pfrom->vProcessMsg.empty()) - return false; - // Just take one message - msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); - pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; - pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman.GetReceiveFloodSize(); - fMoreWork = !pfrom->vProcessMsg.empty(); - } - CNetMessage& msg(msgs.front()); - - msg.SetVersion(pfrom->GetRecvVersion()); - // Scan for message start - if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->id); - pfrom->fDisconnect = true; + std::list<CNetMessage> msgs; + { + LOCK(pfrom->cs_vProcessMsg); + if (pfrom->vProcessMsg.empty()) return false; - } + // Just take one message + msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); + pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; + pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman.GetReceiveFloodSize(); + fMoreWork = !pfrom->vProcessMsg.empty(); + } + CNetMessage& msg(msgs.front()); + + msg.SetVersion(pfrom->GetRecvVersion()); + // Scan for message start + if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { + LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->GetId()); + pfrom->fDisconnect = true; + return false; + } - // Read header - CMessageHeader& hdr = msg.hdr; - if (!hdr.IsValid(chainparams.MessageStart())) - { - LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->id); - return fMoreWork; - } - std::string strCommand = hdr.GetCommand(); + // Read header + CMessageHeader& hdr = msg.hdr; + if (!hdr.IsValid(chainparams.MessageStart())) + { + LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->GetId()); + return fMoreWork; + } + std::string strCommand = hdr.GetCommand(); - // Message size - unsigned int nMessageSize = hdr.nMessageSize; + // Message size + unsigned int nMessageSize = hdr.nMessageSize; - // Checksum - CDataStream& vRecv = msg.vRecv; - const uint256& hash = msg.GetMessageHash(); - if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + // Checksum + CDataStream& vRecv = msg.vRecv; + const uint256& hash = msg.GetMessageHash(); + if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + { + LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, + SanitizeString(strCommand), nMessageSize, + HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), + HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); + return fMoreWork; + } + + // Process message + bool fRet = false; + try + { + fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman, interruptMsgProc); + if (interruptMsgProc) + return false; + if (!pfrom->vRecvGetData.empty()) + fMoreWork = true; + } + catch (const std::ios_base::failure& e) + { + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); + if (strstr(e.what(), "end of data")) { - LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, - SanitizeString(strCommand), nMessageSize, - HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), - HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); - return fMoreWork; + // Allow exceptions from under-length message on vRecv + LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - - // Process message - bool fRet = false; - try + else if (strstr(e.what(), "size too large")) { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman, interruptMsgProc); - if (interruptMsgProc) - return false; - if (!pfrom->vRecvGetData.empty()) - fMoreWork = true; + // Allow exceptions from over-long size + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - catch (const std::ios_base::failure& e) + else if (strstr(e.what(), "non-canonical ReadCompactSize()")) { - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); - if (strstr(e.what(), "end of data")) - { - // Allow exceptions from under-length message on vRecv - LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "size too large")) - { - // Allow exceptions from over-long size - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "non-canonical ReadCompactSize()")) - { - // Allow exceptions from non-canonical encoding - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else - { - PrintExceptionContinue(&e, "ProcessMessages()"); - } + // Allow exceptions from non-canonical encoding + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - catch (const std::exception& e) { + else + { PrintExceptionContinue(&e, "ProcessMessages()"); - } catch (...) { - PrintExceptionContinue(NULL, "ProcessMessages()"); } + } + catch (const std::exception& e) { + PrintExceptionContinue(&e, "ProcessMessages()"); + } catch (...) { + PrintExceptionContinue(NULL, "ProcessMessages()"); + } - if (!fRet) { - LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); - } + if (!fRet) { + LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + } - LOCK(cs_main); - SendRejectsAndCheckIfBanned(pfrom, connman); + LOCK(cs_main); + SendRejectsAndCheckIfBanned(pfrom, connman); return fMoreWork; } @@ -2881,7 +2892,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; - LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); + LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight); connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256())); } } @@ -2911,7 +2922,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); const CBlockIndex *pBestIndex = NULL; // last header queued for delivery - ProcessBlockAvailability(pto->id); // ensure pindexBestKnownBlock is up-to-date + ProcessBlockAvailability(pto->GetId()); // ensure pindexBestKnownBlock is up-to-date if (!fRevertToInv) { bool fFoundStartingHeader = false; @@ -2966,7 +2977,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // We only send up to 1 block as header-and-ids, as otherwise // probably means we're doing an initial-ish-sync or they're slow LogPrint(BCLog::NET, "%s sending header-and-ids %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); + vHeaders.front().GetHash().ToString(), pto->GetId()); int nSendFlags = state.fWantsCmpctWitness ? 0 : SERIALIZE_TRANSACTION_NO_WITNESS; @@ -2974,7 +2985,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr { LOCK(cs_most_recent_block); if (most_recent_block_hash == pBestIndex->GetBlockHash()) { - if (state.fWantsCmpctWitness) + if (state.fWantsCmpctWitness || !fWitnessesPresentInMostRecentCompactBlock) connman.PushMessage(pto, msgMaker.Make(nSendFlags, NetMsgType::CMPCTBLOCK, *most_recent_compact_block)); else { CBlockHeaderAndShortTxIDs cmpctblock(*most_recent_block, state.fWantsCmpctWitness); @@ -2996,10 +3007,10 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr LogPrint(BCLog::NET, "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, vHeaders.size(), vHeaders.front().GetHash().ToString(), - vHeaders.back().GetHash().ToString(), pto->id); + vHeaders.back().GetHash().ToString(), pto->GetId()); } else { LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, - vHeaders.front().GetHash().ToString(), pto->id); + vHeaders.front().GetHash().ToString(), pto->GetId()); } connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); state.pindexBestHeaderSent = pBestIndex; @@ -3028,7 +3039,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr if (!PeerHasHeader(&state, pindex)) { pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); LogPrint(BCLog::NET, "%s: sending inv peer=%d hash=%s\n", __func__, - pto->id, hashToAnnounce.ToString()); + pto->GetId(), hashToAnnounce.ToString()); } } } @@ -3175,7 +3186,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. - LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); + LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->GetId()); pto->fDisconnect = true; return true; } @@ -3188,7 +3199,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); int nOtherPeersWithValidatedDownloads = nPeersWithValidatedDownloads - (state.nBlocksInFlightValidHeaders > 0); if (nNow > state.nDownloadingSince + consensusParams.nPowTargetSpacing * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { - LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->id); + LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->GetId()); pto->fDisconnect = true; return true; } @@ -3203,11 +3214,11 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr NodeId staller = -1; FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller, consensusParams); BOOST_FOREACH(const CBlockIndex *pindex, vToDownload) { - uint32_t nFetchFlags = GetFetchFlags(pto, pindex->pprev, consensusParams); + uint32_t nFetchFlags = GetFetchFlags(pto); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), pindex); LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), - pindex->nHeight, pto->id); + pindex->nHeight, pto->GetId()); } if (state.nBlocksInFlight == 0 && staller != -1) { if (State(staller)->nStallingSince == 0) { @@ -3225,7 +3236,7 @@ bool SendMessages(CNode* pto, CConnman& connman, const std::atomic<bool>& interr const CInv& inv = (*pto->mapAskFor.begin()).second; if (!AlreadyHave(inv)) { - LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->id); + LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->GetId()); vGetData.push_back(inv); if (vGetData.size() >= 1000) { diff --git a/src/netaddress.cpp b/src/netaddress.cpp index ab07270f3f..34a7029862 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -179,12 +179,6 @@ bool CNetAddr::IsLocal() const return false; } -bool CNetAddr::IsMulticast() const -{ - return (IsIPv4() && (GetByte(3) & 0xF0) == 0xE0) - || (GetByte(15) == 0xFF); -} - bool CNetAddr::IsValid() const { // Cleanup 3-byte shifted addresses caused by garbage in size field diff --git a/src/netaddress.h b/src/netaddress.h index a85c2b7452..fbc4d1a65f 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -65,7 +65,6 @@ class CNetAddr bool IsLocal() const; bool IsRoutable() const; bool IsValid() const; - bool IsMulticast() const; enum Network GetNetwork() const; std::string ToString() const; std::string ToStringIP() const; diff --git a/src/netbase.cpp b/src/netbase.cpp index bdc725359a..32557dd179 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -428,18 +428,14 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe if (hSocket == INVALID_SOCKET) return false; - int set = 1; #ifdef SO_NOSIGPIPE + int set = 1; // Different way of disabling SIGPIPE on BSD setsockopt(hSocket, SOL_SOCKET, SO_NOSIGPIPE, (void*)&set, sizeof(int)); #endif //Disable Nagle's algorithm -#ifdef WIN32 - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); -#else - setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (void*)&set, sizeof(int)); -#endif + SetSocketNoDelay(hSocket); // Set to non-blocking if (!SetSocketNonBlocking(hSocket, true)) @@ -672,13 +668,14 @@ std::string NetworkErrorString(int err) std::string NetworkErrorString(int err) { char buf[256]; - const char *s = buf; buf[0] = 0; /* Too bad there are two incompatible implementations of the * thread-safe strerror. */ + const char *s; #ifdef STRERROR_R_CHAR_P /* GNU variant can return a pointer outside the passed buffer */ s = strerror_r(err, buf, sizeof(buf)); #else /* POSIX variant always returns message in buffer */ + s = buf; if (strerror_r(err, buf, sizeof(buf))) buf[0] = 0; #endif @@ -728,6 +725,13 @@ bool SetSocketNonBlocking(SOCKET& hSocket, bool fNonBlocking) return true; } +bool SetSocketNoDelay(SOCKET& hSocket) +{ + int set = 1; + int rc = setsockopt(hSocket, IPPROTO_TCP, TCP_NODELAY, (const char*)&set, sizeof(int)); + return rc == 0; +} + void InterruptSocks5(bool interrupt) { interruptSocks5Recv = interrupt; diff --git a/src/netbase.h b/src/netbase.h index dd33b6e47e..c9d108aadd 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -59,6 +59,8 @@ std::string NetworkErrorString(int err); bool CloseSocket(SOCKET& hSocket); /** Disable or enable blocking-mode for a socket */ bool SetSocketNonBlocking(SOCKET& hSocket, bool fNonBlocking); +/** Set the TCP_NODELAY flag on a socket */ +bool SetSocketNoDelay(SOCKET& hSocket); /** * Convert milliseconds to a struct timeval for e.g. select. */ diff --git a/src/amount.cpp b/src/policy/feerate.cpp index a5f6bc3cd9..a089c02284 100644 --- a/src/amount.cpp +++ b/src/policy/feerate.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "amount.h" +#include "feerate.h" #include "tinyformat.h" diff --git a/src/policy/feerate.h b/src/policy/feerate.h new file mode 100644 index 0000000000..e82268b095 --- /dev/null +++ b/src/policy/feerate.h @@ -0,0 +1,54 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_POLICY_FEERATE_H +#define BITCOIN_POLICY_FEERATE_H + +#include "amount.h" +#include "serialize.h" + +#include <string> + +extern const std::string CURRENCY_UNIT; + +/** + * Fee rate in satoshis per kilobyte: CAmount / kB + */ +class CFeeRate +{ +private: + CAmount nSatoshisPerK; // unit is satoshis-per-1,000-bytes +public: + /** Fee rate of 0 satoshis per kB */ + CFeeRate() : nSatoshisPerK(0) { } + explicit CFeeRate(const CAmount& _nSatoshisPerK): nSatoshisPerK(_nSatoshisPerK) { } + /** Constructor for a fee rate in satoshis per kB. The size in bytes must not exceed (2^63 - 1)*/ + CFeeRate(const CAmount& nFeePaid, size_t nBytes); + CFeeRate(const CFeeRate& other) { nSatoshisPerK = other.nSatoshisPerK; } + /** + * Return the fee in satoshis for the given size in bytes. + */ + CAmount GetFee(size_t nBytes) const; + /** + * Return the fee in satoshis for a size of 1000 bytes + */ + CAmount GetFeePerK() const { return GetFee(1000); } + friend bool operator<(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK < b.nSatoshisPerK; } + friend bool operator>(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK > b.nSatoshisPerK; } + friend bool operator==(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK == b.nSatoshisPerK; } + friend bool operator<=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK <= b.nSatoshisPerK; } + friend bool operator>=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK >= b.nSatoshisPerK; } + CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } + std::string ToString() const; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(nSatoshisPerK); + } +}; + +#endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 38e07dc345..7a9af5edc2 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -7,46 +7,163 @@ #include "policy/policy.h" #include "amount.h" +#include "clientversion.h" #include "primitives/transaction.h" #include "random.h" #include "streams.h" #include "txmempool.h" #include "util.h" -void TxConfirmStats::Initialize(std::vector<double>& defaultBuckets, - unsigned int maxConfirms, double _decay) +static constexpr double INF_FEERATE = 1e99; + +/** + * We will instantiate an instance of this class to track transactions that were + * included in a block. We will lump transactions into a bucket according to their + * approximate feerate and then track how long it took for those txs to be included in a block + * + * The tracking of unconfirmed (mempool) transactions is completely independent of the + * historical tracking of transactions that have been confirmed in a block. + */ +class TxConfirmStats +{ +private: + //Define the buckets we will group transactions into + const std::vector<double>& buckets; // The upper-bound of the range for the bucket (inclusive) + const std::map<double, unsigned int>& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + + // For each bucket X: + // Count the total # of txs in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> txCtAvg; + + // Count the total # of txs confirmed within Y blocks in each bucket + // Track the historical moving average of theses totals over blocks + std::vector<std::vector<double>> confAvg; // confAvg[Y][X] + + // Track moving avg of txs which have been evicted from the mempool + // after failing to be confirmed within Y blocks + std::vector<std::vector<double>> failAvg; // failAvg[Y][X] + + // Sum the total feerate of all tx's in each bucket + // Track the historical moving average of this total over blocks + std::vector<double> avg; + + // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X + // Combine the total value with the tx counts to calculate the avg feerate per bucket + + double decay; + + // Resolution (# of blocks) with which confirmations are tracked + unsigned int scale; + + // Mempool counts of outstanding transactions + // For each bucket X, track the number of transactions in the mempool + // that are unconfirmed for each possible confirmation value Y + std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] + // transactions still unconfirmed after GetMaxConfirms for each bucket + std::vector<int> oldUnconfTxs; + + void resizeInMemoryCounters(size_t newbuckets); + +public: + /** + * Create new TxConfirmStats. This is called by BlockPolicyEstimator's + * constructor with default values. + * @param defaultBuckets contains the upper limits for the bucket boundaries + * @param maxConfirms max number of confirms to track + * @param decay how much to decay the historical moving average per block + */ + TxConfirmStats(const std::vector<double>& defaultBuckets, const std::map<double, unsigned int>& defaultBucketMap, + unsigned int maxPeriods, double decay, unsigned int scale); + + /** Roll the circular buffer for unconfirmed txs*/ + void ClearCurrent(unsigned int nBlockHeight); + + /** + * Record a new transaction data point in the current block stats + * @param blocksToConfirm the number of blocks it took this transaction to confirm + * @param val the feerate of the transaction + * @warning blocksToConfirm is 1-based and has to be >= 1 + */ + void Record(int blocksToConfirm, double val); + + /** Record a new transaction entering the mempool*/ + unsigned int NewTx(unsigned int nBlockHeight, double val); + + /** Remove a transaction from mempool tracking stats*/ + void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, + unsigned int bucketIndex, bool inBlock); + + /** Update our estimates by decaying our historical moving average and updating + with the data gathered from the current block */ + void UpdateMovingAverages(); + + /** + * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets + * to make sure we have enough data points) whose transactions still have sufficient likelihood + * of being confirmed within the target number of confirmations + * @param confTarget target number of confirmations + * @param sufficientTxVal required average number of transactions per block in a bucket range + * @param minSuccess the success probability we require + * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR + * return the highest feerate such that all lower values fail minSuccess + * @param nBlockHeight the current block height + */ + double EstimateMedianVal(int confTarget, double sufficientTxVal, + double minSuccess, bool requireGreater, unsigned int nBlockHeight, + EstimationResult *result = nullptr) const; + + /** Return the max number of confirms we're tracking */ + unsigned int GetMaxConfirms() const { return scale * confAvg.size(); } + + /** Write state of estimation data to a file*/ + void Write(CAutoFile& fileout) const; + + /** + * Read saved state of estimation data from a file and replace all internal data structures and + * variables with this state. + */ + void Read(CAutoFile& filein, int nFileVersion, size_t numBuckets); +}; + + +TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets, + const std::map<double, unsigned int>& defaultBucketMap, + unsigned int maxPeriods, double _decay, unsigned int _scale) + : buckets(defaultBuckets), bucketMap(defaultBucketMap) { decay = _decay; - for (unsigned int i = 0; i < defaultBuckets.size(); i++) { - buckets.push_back(defaultBuckets[i]); - bucketMap[defaultBuckets[i]] = i; - } - confAvg.resize(maxConfirms); - curBlockConf.resize(maxConfirms); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { + scale = _scale; + confAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { confAvg[i].resize(buckets.size()); - curBlockConf[i].resize(buckets.size()); - unconfTxs[i].resize(buckets.size()); + } + failAvg.resize(maxPeriods); + for (unsigned int i = 0; i < maxPeriods; i++) { + failAvg[i].resize(buckets.size()); } - oldUnconfTxs.resize(buckets.size()); - curBlockTxCt.resize(buckets.size()); txCtAvg.resize(buckets.size()); - curBlockVal.resize(buckets.size()); avg.resize(buckets.size()); + + resizeInMemoryCounters(buckets.size()); +} + +void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) { + // newbuckets must be passed in because the buckets referred to during Read have not been updated yet. + unconfTxs.resize(GetMaxConfirms()); + for (unsigned int i = 0; i < unconfTxs.size(); i++) { + unconfTxs[i].resize(newbuckets); + } + oldUnconfTxs.resize(newbuckets); } -// Zero out the data for the current block +// Roll the unconfirmed txs circular buffer void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight) { for (unsigned int j = 0; j < buckets.size(); j++) { oldUnconfTxs[j] += unconfTxs[nBlockHeight%unconfTxs.size()][j]; unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0; - for (unsigned int i = 0; i < curBlockConf.size(); i++) - curBlockConf[i][j] = 0; - curBlockTxCt[j] = 0; - curBlockVal[j] = 0; } } @@ -56,33 +173,38 @@ void TxConfirmStats::Record(int blocksToConfirm, double val) // blocksToConfirm is 1-based if (blocksToConfirm < 1) return; + int periodsToConfirm = (blocksToConfirm + scale - 1)/scale; unsigned int bucketindex = bucketMap.lower_bound(val)->second; - for (size_t i = blocksToConfirm; i <= curBlockConf.size(); i++) { - curBlockConf[i - 1][bucketindex]++; + for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) { + confAvg[i - 1][bucketindex]++; } - curBlockTxCt[bucketindex]++; - curBlockVal[bucketindex] += val; + txCtAvg[bucketindex]++; + avg[bucketindex] += val; } void TxConfirmStats::UpdateMovingAverages() { for (unsigned int j = 0; j < buckets.size(); j++) { for (unsigned int i = 0; i < confAvg.size(); i++) - confAvg[i][j] = confAvg[i][j] * decay + curBlockConf[i][j]; - avg[j] = avg[j] * decay + curBlockVal[j]; - txCtAvg[j] = txCtAvg[j] * decay + curBlockTxCt[j]; + confAvg[i][j] = confAvg[i][j] * decay; + for (unsigned int i = 0; i < failAvg.size(); i++) + failAvg[i][j] = failAvg[i][j] * decay; + avg[j] = avg[j] * decay; + txCtAvg[j] = txCtAvg[j] * decay; } } // returns -1 on error conditions double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, - unsigned int nBlockHeight) + unsigned int nBlockHeight, EstimationResult *result) const { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget double totalNum = 0; // Total number of tx's that were ever confirmed int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget + int periodTarget = (confTarget + scale - 1)/scale; int maxbucketindex = buckets.size() - 1; @@ -105,12 +227,21 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, bool foundAnswer = false; unsigned int bins = unconfTxs.size(); + bool newBucketRange = true; + bool passing = true; + EstimatorBucket passBucket; + EstimatorBucket failBucket; // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { + if (newBucketRange) { + curNearBucket = bucket; + newBucketRange = false; + } curFarBucket = bucket; - nConf += confAvg[confTarget - 1][bucket]; + nConf += confAvg[periodTarget - 1][bucket]; totalNum += txCtAvg[bucket]; + failNum += failAvg[periodTarget - 1][bucket]; for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) extraNum += unconfTxs[(nBlockHeight - confct)%bins][bucket]; extraNum += oldUnconfTxs[bucket]; @@ -119,24 +250,41 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // (Only count the confirmed data points, so that each confirmation count // will be looking at the same amount of data and same bucket breaks) if (totalNum >= sufficientTxVal / (1 - decay)) { - double curPct = nConf / (totalNum + extraNum); + double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate - if (requireGreater && curPct < successBreakPoint) - break; - if (!requireGreater && curPct > successBreakPoint) - break; - + if ((requireGreater && curPct < successBreakPoint) || (!requireGreater && curPct > successBreakPoint)) { + if (passing == true) { + // First time we hit a failure record the failed bucket + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + passing = false; + } + continue; + } // Otherwise update the cumulative stats, and the bucket variables // and reset the counters else { + failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing foundAnswer = true; + passing = true; + passBucket.withinTarget = nConf; nConf = 0; + passBucket.totalConfirmed = totalNum; totalNum = 0; + passBucket.inMempool = extraNum; + passBucket.leftMempool = failNum; + failNum = 0; extraNum = 0; bestNearBucket = curNearBucket; bestFarBucket = curFarBucket; - curNearBucket = bucket + step; + newBucketRange = true; } } } @@ -148,8 +296,8 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // Find the bucket with the median transaction and then report the average feerate from that bucket // This is a compromise between finding the median which we can't since we don't save all tx's // and reporting the average which is less accurate - unsigned int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; - unsigned int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + unsigned int minBucket = std::min(bestNearBucket, bestFarBucket); + unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket); for (unsigned int j = minBucket; j <= maxBucket; j++) { txSum += txCtAvg[j]; } @@ -163,83 +311,109 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, break; } } + + passBucket.start = minBucket ? buckets[minBucket-1] : 0; + passBucket.end = buckets[maxBucket]; } - LogPrint(BCLog::ESTIMATEFEE, "%3d: For conf success %s %4.2f need feerate %s: %12.5g from buckets %8g - %8g Cur Bucket stats %6.2f%% %8.1f/(%.1f+%d mempool)\n", - confTarget, requireGreater ? ">" : "<", successBreakPoint, - requireGreater ? ">" : "<", median, buckets[minBucket], buckets[maxBucket], - 100 * nConf / (totalNum + extraNum), nConf, totalNum, extraNum); + // If we were passing until we reached last few buckets with insufficient data, then report those as failed + if (passing && !newBucketRange) { + unsigned int failMinBucket = std::min(curNearBucket, curFarBucket); + unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket); + failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0; + failBucket.end = buckets[failMaxBucket]; + failBucket.withinTarget = nConf; + failBucket.totalConfirmed = totalNum; + failBucket.inMempool = extraNum; + failBucket.leftMempool = failNum; + } + LogPrint(BCLog::ESTIMATEFEE, "FeeEst: %d %s%.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", + confTarget, requireGreater ? ">" : "<", 100.0 * successBreakPoint, decay, + median, passBucket.start, passBucket.end, + 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool), + passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool, + failBucket.start, failBucket.end, + 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool), + failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool); + + + if (result) { + result->pass = passBucket; + result->fail = failBucket; + result->decay = decay; + result->scale = scale; + } return median; } -void TxConfirmStats::Write(CAutoFile& fileout) +void TxConfirmStats::Write(CAutoFile& fileout) const { fileout << decay; - fileout << buckets; + fileout << scale; fileout << avg; fileout << txCtAvg; fileout << confAvg; + fileout << failAvg; } -void TxConfirmStats::Read(CAutoFile& filein) +void TxConfirmStats::Read(CAutoFile& filein, int nFileVersion, size_t numBuckets) { - // Read data file into temporary variables and do some very basic sanity checking - std::vector<double> fileBuckets; - std::vector<double> fileAvg; - std::vector<std::vector<double> > fileConfAvg; - std::vector<double> fileTxCtAvg; - double fileDecay; - size_t maxConfirms; - size_t numBuckets; - - filein >> fileDecay; - if (fileDecay <= 0 || fileDecay >= 1) - throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - filein >> fileBuckets; - numBuckets = fileBuckets.size(); - if (numBuckets <= 1 || numBuckets > 1000) - throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); - filein >> fileAvg; - if (fileAvg.size() != numBuckets) + // Read data file and do some very basic sanity checking + // buckets and bucketMap are not updated yet, so don't access them + // If there is a read failure, we'll just discard this entire object anyway + size_t maxConfirms, maxPeriods; + + // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor + if (nFileVersion >= 149900) { + filein >> decay; + if (decay <= 0 || decay >= 1) { + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + } + filein >> scale; + } + + filein >> avg; + if (avg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count"); - filein >> fileTxCtAvg; - if (fileTxCtAvg.size() != numBuckets) + } + filein >> txCtAvg; + if (txCtAvg.size() != numBuckets) { throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count"); - filein >> fileConfAvg; - maxConfirms = fileConfAvg.size(); - if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) // one week - throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); - for (unsigned int i = 0; i < maxConfirms; i++) { - if (fileConfAvg[i].size() != numBuckets) - throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); } - // Now that we've processed the entire feerate estimate data file and not - // thrown any errors, we can copy it to our data structures - decay = fileDecay; - buckets = fileBuckets; - avg = fileAvg; - confAvg = fileConfAvg; - txCtAvg = fileTxCtAvg; - bucketMap.clear(); + filein >> confAvg; + maxPeriods = confAvg.size(); + maxConfirms = scale * maxPeriods; - // Resize the current block variables which aren't stored in the data file - // to match the number of confirms and buckets - curBlockConf.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - curBlockConf[i].resize(buckets.size()); + if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week + throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (confAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count"); + } } - curBlockTxCt.resize(buckets.size()); - curBlockVal.resize(buckets.size()); - unconfTxs.resize(maxConfirms); - for (unsigned int i = 0; i < maxConfirms; i++) { - unconfTxs[i].resize(buckets.size()); + if (nFileVersion >= 149900) { + filein >> failAvg; + if (maxPeriods != failAvg.size()) { + throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures"); + } + for (unsigned int i = 0; i < maxPeriods; i++) { + if (failAvg[i].size() != numBuckets) { + throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts"); + } + } + } else { + failAvg.resize(confAvg.size()); + for (unsigned int i = 0; i < failAvg.size(); i++) { + failAvg[i].resize(numBuckets); + } } - oldUnconfTxs.resize(buckets.size()); - for (unsigned int i = 0; i < buckets.size(); i++) - bucketMap[buckets[i]] = i; + // Resize the current block variables which aren't stored in the data file + // to match the number of confirms and buckets + resizeInMemoryCounters(numBuckets); LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n", numBuckets, maxConfirms); @@ -253,7 +427,7 @@ unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val) return bucketindex; } -void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex) +void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock) { //nBestSeenHeight is not updated yet for the new block int blocksAgo = nBestSeenHeight - entryHeight; @@ -281,6 +455,12 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe blockIndex, bucketindex); } } + if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period + unsigned int periodsAgo = blocksAgo / scale; + for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) { + failAvg[i][bucketindex]++; + } + } } // This function is called from CTxMemPool::removeUnchecked to ensure @@ -288,11 +468,14 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe // tracked. Txs that were part of a block have already been removed in // processBlockTx to ensure they are never double tracked, but it is // of no harm to try to remove them again. -bool CBlockPolicyEstimator::removeTx(uint256 hash) +bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock) { + LOCK(cs_feeEstimator); std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash); if (pos != mapMemPoolTxs.end()) { - feeStats.removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex); + feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); + longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); mapMemPoolTxs.erase(hash); return true; } else { @@ -301,20 +484,33 @@ bool CBlockPolicyEstimator::removeTx(uint256 hash) } CBlockPolicyEstimator::CBlockPolicyEstimator() - : nBestSeenHeight(0), trackedTxs(0), untrackedTxs(0) + : nBestSeenHeight(0), firstRecordedHeight(0), historicalFirst(0), historicalBest(0), trackedTxs(0), untrackedTxs(0) { static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero"); - minTrackedFee = CFeeRate(MIN_BUCKET_FEERATE); - std::vector<double> vfeelist; - for (double bucketBoundary = minTrackedFee.GetFeePerK(); bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING) { - vfeelist.push_back(bucketBoundary); + size_t bucketIndex = 0; + for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) { + buckets.push_back(bucketBoundary); + bucketMap[bucketBoundary] = bucketIndex; } - vfeelist.push_back(INF_FEERATE); - feeStats.Initialize(vfeelist, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY); + buckets.push_back(INF_FEERATE); + bucketMap[INF_FEERATE] = bucketIndex; + assert(bucketMap.size() == buckets.size()); + + feeStats = new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE); + shortStats = new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE); + longStats = new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE); +} + +CBlockPolicyEstimator::~CBlockPolicyEstimator() +{ + delete feeStats; + delete shortStats; + delete longStats; } void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) { + LOCK(cs_feeEstimator); unsigned int txHeight = entry.GetHeight(); uint256 hash = entry.GetTx().GetHash(); if (mapMemPoolTxs.count(hash)) { @@ -343,12 +539,17 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); mapMemPoolTxs[hash].blockHeight = txHeight; - mapMemPoolTxs[hash].bucketIndex = feeStats.NewTx(txHeight, (double)feeRate.GetFeePerK()); + unsigned int bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + mapMemPoolTxs[hash].bucketIndex = bucketIndex; + unsigned int bucketIndex2 = shortStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex2); + unsigned int bucketIndex3 = longStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + assert(bucketIndex == bucketIndex3); } bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) { - if (!removeTx(entry->GetTx().GetHash())) { + if (!removeTx(entry->GetTx().GetHash(), true)) { // This transaction wasn't being tracked for fee estimation return false; } @@ -367,13 +568,16 @@ bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM // Feerates are stored and reported as BTC-per-kb: CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); - feeStats.Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + shortStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + longStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); return true; } void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, std::vector<const CTxMemPoolEntry*>& entries) { + LOCK(cs_feeEstimator); if (nBlockHeight <= nBestSeenHeight) { // Ignore side chains and re-orgs; assuming they are random // they don't affect the estimate. @@ -388,34 +592,77 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, // of unconfirmed txs to remove from tracking. nBestSeenHeight = nBlockHeight; - // Clear the current block state and update unconfirmed circular buffer - feeStats.ClearCurrent(nBlockHeight); + // Update unconfirmed circular buffer + feeStats->ClearCurrent(nBlockHeight); + shortStats->ClearCurrent(nBlockHeight); + longStats->ClearCurrent(nBlockHeight); + + // Decay all exponential averages + feeStats->UpdateMovingAverages(); + shortStats->UpdateMovingAverages(); + longStats->UpdateMovingAverages(); unsigned int countedTxs = 0; - // Repopulate the current block states - for (unsigned int i = 0; i < entries.size(); i++) { - if (processBlockTx(nBlockHeight, entries[i])) + // Update averages with data points from current block + for (const auto& entry : entries) { + if (processBlockTx(nBlockHeight, entry)) countedTxs++; } - // Update all exponential averages with the current block state - feeStats.UpdateMovingAverages(); + if (firstRecordedHeight == 0 && countedTxs > 0) { + firstRecordedHeight = nBestSeenHeight; + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight); + } + - LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy after updating estimates for %u of %u txs in block, since last block %u of %u tracked, new mempool map size %u\n", - countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size()); + LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n", + countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(), + MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current"); trackedTxs = 0; untrackedTxs = 0; } -CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) +CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) const { - // Return failure if trying to analyze a target we're not tracking // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) + if (confTarget <= 1) + return CFeeRate(0); + + return estimateRawFee(confTarget, DOUBLE_SUCCESS_PCT, FeeEstimateHorizon::MED_HALFLIFE); +} + +CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const +{ + TxConfirmStats* stats; + double sufficientTxs = SUFFICIENT_FEETXS; + switch (horizon) { + case FeeEstimateHorizon::SHORT_HALFLIFE: { + stats = shortStats; + sufficientTxs = SUFFICIENT_TXS_SHORT; + break; + } + case FeeEstimateHorizon::MED_HALFLIFE: { + stats = feeStats; + break; + } + case FeeEstimateHorizon::LONG_HALFLIFE: { + stats = longStats; + break; + } + default: { return CFeeRate(0); + } + } - double median = feeStats.EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); + LOCK(cs_feeEstimator); + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms()) + return CFeeRate(0); + if (successThreshold > 1) + return CFeeRate(0); + + double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, true, nBestSeenHeight, result); if (median < 0) return CFeeRate(0); @@ -423,25 +670,148 @@ CFeeRate CBlockPolicyEstimator::estimateFee(int confTarget) return CFeeRate(median); } -CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool) +unsigned int CBlockPolicyEstimator::BlockSpan() const +{ + if (firstRecordedHeight == 0) return 0; + assert(nBestSeenHeight >= firstRecordedHeight); + + return nBestSeenHeight - firstRecordedHeight; +} + +unsigned int CBlockPolicyEstimator::HistoricalBlockSpan() const +{ + if (historicalFirst == 0) return 0; + assert(historicalBest >= historicalFirst); + + if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0; + + return historicalBest - historicalFirst; +} + +unsigned int CBlockPolicyEstimator::MaxUsableEstimate() const +{ + // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate + return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2); +} + +/** Return a fee estimate at the required successThreshold from the shortest + * time horizon which tracks confirmations up to the desired target. If + * checkShorterHorizon is requested, also allow short time horizon estimates + * for a lower target to reduce the given answer */ +double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const +{ + double estimate = -1; + if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) { + // Find estimate from shortest time horizon possible + if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon + estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + } + else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon + estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + else { // long horizon + estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + } + if (checkShorterHorizon) { + // If a lower confTarget from a more recent horizon returns a lower answer use it. + if (confTarget > feeStats->GetMaxConfirms()) { + double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, true, nBestSeenHeight); + if (medMax > 0 && (estimate == -1 || medMax < estimate)) + estimate = medMax; + } + if (confTarget > shortStats->GetMaxConfirms()) { + double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, true, nBestSeenHeight); + if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) + estimate = shortMax; + } + } + } + return estimate; +} + +/** Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met + * at 2 * target for any longer time horizons. + */ +double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget) const +{ + double estimate = -1; + if (doubleTarget <= shortStats->GetMaxConfirms()) { + estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + } + if (doubleTarget <= feeStats->GetMaxConfirms()) { + double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, true, nBestSeenHeight); + if (longEstimate > estimate) { + estimate = longEstimate; + } + } + return estimate; +} + +/** estimateSmartFee returns the max of the feerates calculated with a 60% + * threshold required at target / 2, an 85% threshold required at target and a + * 95% threshold required at 2 * target. Each calculation is performed at the + * shortest time horizon which tracks the required target. Conservative + * estimates, however, required the 95% threshold at 2 * target be met for any + * longer time horizons also. + */ +CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative) const { if (answerFoundAtTarget) *answerFoundAtTarget = confTarget; - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || (unsigned int)confTarget > feeStats.GetMaxConfirms()) - return CFeeRate(0); - - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget == 1) - confTarget = 2; double median = -1; - while (median < 0 && (unsigned int)confTarget <= feeStats.GetMaxConfirms()) { - median = feeStats.EstimateMedianVal(confTarget++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, nBestSeenHeight); - } + { + LOCK(cs_feeEstimator); + + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) + return CFeeRate(0); + + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget == 1) + confTarget = 2; + + unsigned int maxUsableEstimate = MaxUsableEstimate(); + if (maxUsableEstimate <= 1) + return CFeeRate(0); + + if ((unsigned int)confTarget > maxUsableEstimate) { + confTarget = maxUsableEstimate; + } + + assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints + + /** true is passed to estimateCombined fee for target/2 and target so + * that we check the max confirms for shorter time horizons as well. + * This is necessary to preserve monotonically increasing estimates. + * For non-conservative estimates we do the same thing for 2*target, but + * for conservative estimates we want to skip these shorter horizons + * checks for 2*target because we are taking the max over all time + * horizons so we already have monotonically increasing estimates and + * the purpose of conservative estimates is not to let short term + * fluctuations lower our estimates by too much. + */ + double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true); + double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true); + double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative); + median = halfEst; + if (actualEst > median) { + median = actualEst; + } + if (doubleEst > median) { + median = doubleEst; + } + + if (conservative || median == -1) { + double consEst = estimateConservativeFee(2 * confTarget); + if (consEst > median) { + median = consEst; + } + } + } // Must unlock cs_feeEstimator before taking mempool locks if (answerFoundAtTarget) - *answerFoundAtTarget = confTarget - 1; + *answerFoundAtTarget = confTarget; // If mempool is limiting txs , return at least the min feerate from the mempool CAmount minPoolFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); @@ -454,26 +824,134 @@ CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, int *answerFoun return CFeeRate(median); } -void CBlockPolicyEstimator::Write(CAutoFile& fileout) + +bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const { - fileout << nBestSeenHeight; - feeStats.Write(fileout); + try { + LOCK(cs_feeEstimator); + fileout << 149900; // version required to read: 0.14.99 or later + fileout << CLIENT_VERSION; // version that wrote the file + fileout << nBestSeenHeight; + if (BlockSpan() > HistoricalBlockSpan()/2) { + fileout << firstRecordedHeight << nBestSeenHeight; + } + else { + fileout << historicalFirst << historicalBest; + } + fileout << buckets; + feeStats->Write(fileout); + shortStats->Write(fileout); + longStats->Write(fileout); + } + catch (const std::exception&) { + LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n"); + return false; + } + return true; } -void CBlockPolicyEstimator::Read(CAutoFile& filein, int nFileVersion) +bool CBlockPolicyEstimator::Read(CAutoFile& filein) { - int nFileBestSeenHeight; - filein >> nFileBestSeenHeight; - feeStats.Read(filein); - nBestSeenHeight = nFileBestSeenHeight; - // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + try { + LOCK(cs_feeEstimator); + int nVersionRequired, nVersionThatWrote; + unsigned int nFileBestSeenHeight, nFileHistoricalFirst, nFileHistoricalBest; + filein >> nVersionRequired >> nVersionThatWrote; + if (nVersionRequired > CLIENT_VERSION) + return error("CBlockPolicyEstimator::Read(): up-version (%d) fee estimate file", nVersionRequired); + + // Read fee estimates file into temporary variables so existing data + // structures aren't corrupted if there is an exception. + filein >> nFileBestSeenHeight; + + if (nVersionThatWrote < 149900) { + // Read the old fee estimates file for temporary use, but then discard. Will start collecting data from scratch. + // decay is stored before buckets in old versions, so pre-read decay and pass into TxConfirmStats constructor + double tempDecay; + filein >> tempDecay; + if (tempDecay <= 0 || tempDecay >= 1) + throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + + std::vector<double> tempBuckets; + filein >> tempBuckets; + size_t tempNum = tempBuckets.size(); + if (tempNum <= 1 || tempNum > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::map<double, unsigned int> tempMap; + + std::unique_ptr<TxConfirmStats> tempFeeStats(new TxConfirmStats(tempBuckets, tempMap, MED_BLOCK_PERIODS, tempDecay, 1)); + tempFeeStats->Read(filein, nVersionThatWrote, tempNum); + // if nVersionThatWrote < 139900 then another TxConfirmStats (for priority) follows but can be ignored. + + tempMap.clear(); + for (unsigned int i = 0; i < tempBuckets.size(); i++) { + tempMap[tempBuckets[i]] = i; + } + } + else { // nVersionThatWrote >= 149900 + filein >> nFileHistoricalFirst >> nFileHistoricalBest; + if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) { + throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid"); + } + std::vector<double> fileBuckets; + filein >> fileBuckets; + size_t numBuckets = fileBuckets.size(); + if (numBuckets <= 1 || numBuckets > 1000) + throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + std::unique_ptr<TxConfirmStats> fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE)); + std::unique_ptr<TxConfirmStats> fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE)); + std::unique_ptr<TxConfirmStats> fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE)); + fileFeeStats->Read(filein, nVersionThatWrote, numBuckets); + fileShortStats->Read(filein, nVersionThatWrote, numBuckets); + fileLongStats->Read(filein, nVersionThatWrote, numBuckets); + + // Fee estimates file parsed correctly + // Copy buckets from file and refresh our bucketmap + buckets = fileBuckets; + bucketMap.clear(); + for (unsigned int i = 0; i < buckets.size(); i++) { + bucketMap[buckets[i]] = i; + } + + // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap + delete feeStats; + delete shortStats; + delete longStats; + feeStats = fileFeeStats.release(); + shortStats = fileShortStats.release(); + longStats = fileLongStats.release(); + + nBestSeenHeight = nFileBestSeenHeight; + historicalFirst = nFileHistoricalFirst; + historicalBest = nFileHistoricalBest; + } + } + catch (const std::exception& e) { + LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what()); + return false; + } + return true; +} + +void CBlockPolicyEstimator::FlushUnconfirmed(CTxMemPool& pool) { + int64_t startclear = GetTimeMicros(); + std::vector<uint256> txids; + pool.queryHashes(txids); + LOCK(cs_feeEstimator); + for (auto& txid : txids) { + removeTx(txid, false); + } + int64_t endclear = GetTimeMicros(); + LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n",txids.size(), (endclear - startclear)*0.000001); } FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) { CAmount minFeeLimit = std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2); feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING) { + for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_FILTER_FEERATE; bucketBoundary *= FEE_FILTER_SPACING) { feeset.insert(bucketBoundary); } } diff --git a/src/policy/fees.h b/src/policy/fees.h index dd01c90c45..e99fec2c39 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -6,8 +6,10 @@ #define BITCOIN_POLICYESTIMATOR_H #include "amount.h" +#include "feerate.h" #include "uint256.h" #include "random.h" +#include "sync.h" #include <map> #include <string> @@ -17,6 +19,7 @@ class CAutoFile; class CFeeRate; class CTxMemPoolEntry; class CTxMemPool; +class TxConfirmStats; /** \class CBlockPolicyEstimator * The BlockPolicyEstimator is used for estimating the feerate needed @@ -39,161 +42,58 @@ class CTxMemPool; * within your desired 5 blocks. * * Here is a brief description of the implementation: - * When a transaction enters the mempool, we - * track the height of the block chain at entry. Whenever a block comes in, - * we count the number of transactions in each bucket and the total amount of feerate - * paid in each bucket. Then we calculate how many blocks Y it took each - * transaction to be mined and we track an array of counters in each bucket - * for how long it to took transactions to get confirmed from 1 to a max of 25 - * and we increment all the counters from Y up to 25. This is because for any - * number Z>=Y the transaction was successfully mined within Z blocks. We - * want to save a history of this information, so at any time we have a - * counter of the total number of transactions that happened in a given feerate - * bucket and the total number that were confirmed in each number 1-25 blocks - * or less for any bucket. We save this history by keeping an exponentially - * decaying moving average of each one of these stats. Furthermore we also - * keep track of the number unmined (in mempool) transactions in each bucket - * and for how many blocks they have been outstanding and use that to increase - * the number of transactions we've seen in that feerate bucket when calculating - * an estimate for any number of confirmations below the number of blocks - * they've been outstanding. + * When a transaction enters the mempool, we track the height of the block chain + * at entry. All further calculations are conducted only on this set of "seen" + * transactions. Whenever a block comes in, we count the number of transactions + * in each bucket and the total amount of feerate paid in each bucket. Then we + * calculate how many blocks Y it took each transaction to be mined. We convert + * from a number of blocks to a number of periods Y' each encompassing "scale" + * blocks. This is is tracked in 3 different data sets each up to a maximum + * number of periods. Within each data set we have an array of counters in each + * feerate bucket and we increment all the counters from Y' up to max periods + * representing that a tx was successfully confirmed in less than or equal to + * that many periods. We want to save a history of this information, so at any + * time we have a counter of the total number of transactions that happened in a + * given feerate bucket and the total number that were confirmed in each of the + * periods or less for any bucket. We save this history by keeping an + * exponentially decaying moving average of each one of these stats. This is + * done for a different decay in each of the 3 data sets to keep relevant data + * from different time horizons. Furthermore we also keep track of the number + * unmined (in mempool or left mempool without being included in a block) + * transactions in each bucket and for how many blocks they have been + * outstanding and use both of these numbers to increase the number of transactions + * we've seen in that feerate bucket when calculating an estimate for any number + * of confirmations below the number of blocks they've been outstanding. */ -/** - * We will instantiate an instance of this class to track transactions that were - * included in a block. We will lump transactions into a bucket according to their - * approximate feerate and then track how long it took for those txs to be included in a block - * - * The tracking of unconfirmed (mempool) transactions is completely independent of the - * historical tracking of transactions that have been confirmed in a block. - */ -class TxConfirmStats -{ -private: - //Define the buckets we will group transactions into - std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) - std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket - - // For each bucket X: - // Count the total # of txs in each bucket - // Track the historical moving average of this total over blocks - std::vector<double> txCtAvg; - // and calculate the total for the current block to update the moving average - std::vector<int> curBlockTxCt; - - // Count the total # of txs confirmed within Y blocks in each bucket - // Track the historical moving average of theses totals over blocks - std::vector<std::vector<double> > confAvg; // confAvg[Y][X] - // and calculate the totals for the current block to update the moving averages - std::vector<std::vector<int> > curBlockConf; // curBlockConf[Y][X] - - // Sum the total feerate of all tx's in each bucket - // Track the historical moving average of this total over blocks - std::vector<double> avg; - // and calculate the total for the current block to update the moving average - std::vector<double> curBlockVal; +/* Identifier for each of the 3 different TxConfirmStats which will track + * history over different time horizons. */ +enum FeeEstimateHorizon { + SHORT_HALFLIFE = 0, + MED_HALFLIFE = 1, + LONG_HALFLIFE = 2 +}; - // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X - // Combine the total value with the tx counts to calculate the avg feerate per bucket +/* Used to return detailed information about a feerate bucket */ +struct EstimatorBucket +{ + double start = -1; + double end = -1; + double withinTarget = 0; + double totalConfirmed = 0; + double inMempool = 0; + double leftMempool = 0; +}; +/* Used to return detailed information about a fee estimate calculation */ +struct EstimationResult +{ + EstimatorBucket pass; + EstimatorBucket fail; double decay; - - // Mempool counts of outstanding transactions - // For each bucket X, track the number of transactions in the mempool - // that are unconfirmed for each possible confirmation value Y - std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X] - // transactions still unconfirmed after MAX_CONFIRMS for each bucket - std::vector<int> oldUnconfTxs; - -public: - /** - * Initialize the data structures. This is called by BlockPolicyEstimator's - * constructor with default values. - * @param defaultBuckets contains the upper limits for the bucket boundaries - * @param maxConfirms max number of confirms to track - * @param decay how much to decay the historical moving average per block - */ - void Initialize(std::vector<double>& defaultBuckets, unsigned int maxConfirms, double decay); - - /** Clear the state of the curBlock variables to start counting for the new block */ - void ClearCurrent(unsigned int nBlockHeight); - - /** - * Record a new transaction data point in the current block stats - * @param blocksToConfirm the number of blocks it took this transaction to confirm - * @param val the feerate of the transaction - * @warning blocksToConfirm is 1-based and has to be >= 1 - */ - void Record(int blocksToConfirm, double val); - - /** Record a new transaction entering the mempool*/ - unsigned int NewTx(unsigned int nBlockHeight, double val); - - /** Remove a transaction from mempool tracking stats*/ - void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, - unsigned int bucketIndex); - - /** Update our estimates by decaying our historical moving average and updating - with the data gathered from the current block */ - void UpdateMovingAverages(); - - /** - * Calculate a feerate estimate. Find the lowest value bucket (or range of buckets - * to make sure we have enough data points) whose transactions still have sufficient likelihood - * of being confirmed within the target number of confirmations - * @param confTarget target number of confirmations - * @param sufficientTxVal required average number of transactions per block in a bucket range - * @param minSuccess the success probability we require - * @param requireGreater return the lowest feerate such that all higher values pass minSuccess OR - * return the highest feerate such that all lower values fail minSuccess - * @param nBlockHeight the current block height - */ - double EstimateMedianVal(int confTarget, double sufficientTxVal, - double minSuccess, bool requireGreater, unsigned int nBlockHeight); - - /** Return the max number of confirms we're tracking */ - unsigned int GetMaxConfirms() { return confAvg.size(); } - - /** Write state of estimation data to a file*/ - void Write(CAutoFile& fileout); - - /** - * Read saved state of estimation data from a file and replace all internal data structures and - * variables with this state. - */ - void Read(CAutoFile& filein); + unsigned int scale; }; - - -/** Track confirm delays up to 25 blocks, can't estimate beyond that */ -static const unsigned int MAX_BLOCK_CONFIRMS = 25; - -/** Decay of .998 is a half-life of 346 blocks or about 2.4 days */ -static const double DEFAULT_DECAY = .998; - -/** Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough */ -static const double MIN_SUCCESS_PCT = .95; - -/** Require an avg of 1 tx in the combined feerate bucket per block to have stat significance */ -static const double SUFFICIENT_FEETXS = 1; - -// Minimum and Maximum values for tracking feerates -// The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we -// might ever want to track. Historically this has been 1000 since it was -// inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it -// invalidates old estimates files. So leave it at 1000 unless it becomes -// necessary to lower it, and then lower it substantially. -static constexpr double MIN_BUCKET_FEERATE = 1000; -static const double MAX_BUCKET_FEERATE = 1e7; -static const double INF_FEERATE = MAX_MONEY; - -// We have to lump transactions into buckets based on feerate, but we want to be able -// to give accurate estimates over a large range of potential feerates -// Therefore it makes sense to exponentially space the buckets -/** Spacing of FeeRate buckets */ -static const double FEE_SPACING = 1.1; - /** * We want to be able to estimate feerates that are needed on tx's to be included in * a certain number of blocks. Every time a block is added to the best chain, this class records @@ -201,41 +101,101 @@ static const double FEE_SPACING = 1.1; */ class CBlockPolicyEstimator { +private: + /** Track confirm delays up to 12 blocks for short horizon */ + static constexpr unsigned int SHORT_BLOCK_PERIODS = 12; + static constexpr unsigned int SHORT_SCALE = 1; + /** Track confirm delays up to 48 blocks for medium horizon */ + static constexpr unsigned int MED_BLOCK_PERIODS = 24; + static constexpr unsigned int MED_SCALE = 2; + /** Track confirm delays up to 1008 blocks for long horizon */ + static constexpr unsigned int LONG_BLOCK_PERIODS = 42; + static constexpr unsigned int LONG_SCALE = 24; + /** Historical estimates that are older than this aren't valid */ + static const unsigned int OLDEST_ESTIMATE_HISTORY = 6 * 1008; + + /** Decay of .962 is a half-life of 18 blocks or about 3 hours */ + static constexpr double SHORT_DECAY = .962; + /** Decay of .998 is a half-life of 144 blocks or about 1 day */ + static constexpr double MED_DECAY = .9952; + /** Decay of .9995 is a half-life of 1008 blocks or about 1 week */ + static constexpr double LONG_DECAY = .99931; + + /** Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks*/ + static constexpr double HALF_SUCCESS_PCT = .6; + /** Require greater than 85% of X feerate transactions to be confirmed within Y blocks*/ + static constexpr double SUCCESS_PCT = .85; + /** Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks*/ + static constexpr double DOUBLE_SUCCESS_PCT = .95; + + /** Require an avg of 0.1 tx in the combined feerate bucket per block to have stat significance */ + static constexpr double SUFFICIENT_FEETXS = 0.1; + /** Require an avg of 0.5 tx when using short decay since there are fewer blocks considered*/ + static constexpr double SUFFICIENT_TXS_SHORT = 0.5; + + /** Minimum and Maximum values for tracking feerates + * The MIN_BUCKET_FEERATE should just be set to the lowest reasonable feerate we + * might ever want to track. Historically this has been 1000 since it was + * inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it + * invalidates old estimates files. So leave it at 1000 unless it becomes + * necessary to lower it, and then lower it substantially. + */ + static constexpr double MIN_BUCKET_FEERATE = 1000; + static constexpr double MAX_BUCKET_FEERATE = 1e7; + + /** Spacing of FeeRate buckets + * We have to lump transactions into buckets based on feerate, but we want to be able + * to give accurate estimates over a large range of potential feerates + * Therefore it makes sense to exponentially space the buckets + */ + static constexpr double FEE_SPACING = 1.05; + public: /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ CBlockPolicyEstimator(); + ~CBlockPolicyEstimator(); /** Process all the transactions that have been included in a block */ void processBlock(unsigned int nBlockHeight, std::vector<const CTxMemPoolEntry*>& entries); - /** Process a transaction confirmed in a block*/ - bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); - /** Process a transaction accepted to the mempool*/ void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate); /** Remove a transaction from the mempool tracking stats*/ - bool removeTx(uint256 hash); + bool removeTx(uint256 hash, bool inBlock); + + /** DEPRECATED. Return a feerate estimate */ + CFeeRate estimateFee(int confTarget) const; - /** Return a feerate estimate */ - CFeeRate estimateFee(int confTarget); + /** Estimate feerate needed to get be included in a block within confTarget + * blocks. If no answer can be given at confTarget, return an estimate at + * the closest target where one can be given. 'conservative' estimates are + * valid over longer time horizons also. + */ + CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool, bool conservative = true) const; - /** Estimate feerate needed to get be included in a block within - * confTarget blocks. If no answer can be given at confTarget, return an - * estimate at the lowest target where one can be given. + /** Return a specific fee estimate calculation with a given success + * threshold and time horizon, and optionally return detailed data about + * calculation */ - CFeeRate estimateSmartFee(int confTarget, int *answerFoundAtTarget, const CTxMemPool& pool); + CFeeRate estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult *result = nullptr) const; /** Write estimation data to a file */ - void Write(CAutoFile& fileout); + bool Write(CAutoFile& fileout) const; /** Read estimation data from a file */ - void Read(CAutoFile& filein, int nFileVersion); + bool Read(CAutoFile& filein); + + /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ + void FlushUnconfirmed(CTxMemPool& pool); private: - CFeeRate minTrackedFee; //!< Passed to constructor to avoid dependency on main unsigned int nBestSeenHeight; + unsigned int firstRecordedHeight; + unsigned int historicalFirst; + unsigned int historicalBest; + struct TxStatsInfo { unsigned int blockHeight; @@ -247,14 +207,43 @@ private: std::map<uint256, TxStatsInfo> mapMemPoolTxs; /** Classes to track historical data on transaction confirmations */ - TxConfirmStats feeStats; + TxConfirmStats* feeStats; + TxConfirmStats* shortStats; + TxConfirmStats* longStats; unsigned int trackedTxs; unsigned int untrackedTxs; + + std::vector<double> buckets; // The upper-bound of the range for the bucket (inclusive) + std::map<double, unsigned int> bucketMap; // Map of bucket upper-bound to index into all vectors by bucket + + mutable CCriticalSection cs_feeEstimator; + + /** Process a transaction confirmed in a block*/ + bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry); + + /** Helper for estimateSmartFee */ + double estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon) const; + /** Helper for estimateSmartFee */ + double estimateConservativeFee(unsigned int doubleTarget) const; + /** Number of blocks of data recorded while fee estimates have been running */ + unsigned int BlockSpan() const; + /** Number of blocks of recorded fee estimate data represented in saved data file */ + unsigned int HistoricalBlockSpan() const; + /** Calculation of highest target that reasonable estimate can be provided for */ + unsigned int MaxUsableEstimate() const; }; class FeeFilterRounder { +private: + static constexpr double MAX_FILTER_FEERATE = 1e7; + /** FEE_FILTER_SPACING is just used to provide some quantization of fee + * filter results. Historically it reused FEE_SPACING, but it is completely + * unrelated, and was made a separate constant so the two concepts are not + * tied together */ + static constexpr double FEE_FILTER_SPACING = 1.1; + public: /** Create new FeeFilterRounder */ FeeFilterRounder(const CFeeRate& minIncrementalFee); @@ -266,4 +255,5 @@ private: std::set<double> feeset; FastRandomContext insecure_rand; }; + #endif /*BITCOIN_POLICYESTIMATOR_H */ diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index ec398f6627..14d58e7442 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -8,12 +8,50 @@ #include "policy/policy.h" #include "validation.h" +#include "coins.h" #include "tinyformat.h" #include "util.h" #include "utilstrencodings.h" #include <boost/foreach.hpp> +CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) +{ + // "Dust" is defined in terms of dustRelayFee, + // which has units satoshis-per-kilobyte. + // If you'd pay more than 1/3 in fees + // to spend something, then we consider it dust. + // A typical spendable non-segwit txout is 34 bytes big, and will + // need a CTxIn of at least 148 bytes to spend: + // so dust is a spendable txout less than + // 546*dustRelayFee/1000 (in satoshis). + // A typical spendable segwit txout is 31 bytes big, and will + // need a CTxIn of at least 67 bytes to spend: + // so dust is a spendable txout less than + // 294*dustRelayFee/1000 (in satoshis). + if (txout.scriptPubKey.IsUnspendable()) + return 0; + + size_t nSize = GetSerializeSize(txout, SER_DISK, 0); + int witnessversion = 0; + std::vector<unsigned char> witnessprogram; + + if (txout.scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { + // sum the sizes of the parts of a transaction input + // with 75% segwit discount applied to the script size. + nSize += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); + } else { + nSize += (32 + 4 + 1 + 107 + 4); // the 148 mentioned above + } + + return 3 * dustRelayFeeIn.GetFee(nSize); +} + +bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) +{ + return (txout.nValue < GetDustThreshold(txout, dustRelayFeeIn)); +} + /** * Check transaction inputs to mitigate two * potential denial-of-service attacks: @@ -105,7 +143,7 @@ bool IsStandardTx(const CTransaction& tx, std::string& reason, const bool witnes else if ((whichType == TX_MULTISIG) && (!fIsBareMultisigStd)) { reason = "bare-multisig"; return false; - } else if (txout.IsDust(dustRelayFee)) { + } else if (IsDust(txout, ::dustRelayFee)) { reason = "dust"; return false; } @@ -127,7 +165,7 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxOut& prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut& prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; std::vector<std::vector<unsigned char> > vSolutions; txnouttype whichType; @@ -166,7 +204,7 @@ bool IsWitnessStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) if (tx.vin[i].scriptWitness.IsNull()) continue; - const CTxOut &prev = mapInputs.GetOutputFor(tx.vin[i]); + const CTxOut &prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; // get the scriptPubKey corresponding to this input: CScript prevScript = prev.scriptPubKey; diff --git a/src/policy/policy.h b/src/policy/policy.h index 6df541bc0f..2c2ea9d5b8 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -7,12 +7,14 @@ #define BITCOIN_POLICY_POLICY_H #include "consensus/consensus.h" +#include "feerate.h" #include "script/interpreter.h" #include "script/standard.h" #include <string> class CCoinsViewCache; +class CTxOut; /** Default for -blockmaxsize, which controls the maximum size of block the mining code will create **/ static const unsigned int DEFAULT_BLOCK_MAX_SIZE = 750000; @@ -72,6 +74,10 @@ static const unsigned int STANDARD_NOT_MANDATORY_VERIFY_FLAGS = STANDARD_SCRIPT_ static const unsigned int STANDARD_LOCKTIME_VERIFY_FLAGS = LOCKTIME_VERIFY_SEQUENCE | LOCKTIME_MEDIAN_TIME_PAST; +CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee); + +bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee); + bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType, const bool witnessEnabled = false); /** * Check for standard transaction types diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index d413e8b087..00ac0b92b5 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -22,8 +22,8 @@ public: uint256 hash; uint32_t n; - COutPoint() { SetNull(); } - COutPoint(uint256 hashIn, uint32_t nIn) { hash = hashIn; n = nIn; } + COutPoint(): n((uint32_t) -1) { } + COutPoint(const uint256& hashIn, uint32_t nIn): hash(hashIn), n(nIn) { } ADD_SERIALIZE_METHODS; @@ -161,43 +161,6 @@ public: return (nValue == -1); } - CAmount GetDustThreshold(const CFeeRate &minRelayTxFee) const - { - // "Dust" is defined in terms of CTransaction::minRelayTxFee, - // which has units satoshis-per-kilobyte. - // If you'd pay more than 1/3 in fees - // to spend something, then we consider it dust. - // A typical spendable non-segwit txout is 34 bytes big, and will - // need a CTxIn of at least 148 bytes to spend: - // so dust is a spendable txout less than - // 546*minRelayTxFee/1000 (in satoshis). - // A typical spendable segwit txout is 31 bytes big, and will - // need a CTxIn of at least 67 bytes to spend: - // so dust is a spendable txout less than - // 294*minRelayTxFee/1000 (in satoshis). - if (scriptPubKey.IsUnspendable()) - return 0; - - size_t nSize = GetSerializeSize(*this, SER_DISK, 0); - int witnessversion = 0; - std::vector<unsigned char> witnessprogram; - - if (scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { - // sum the sizes of the parts of a transaction input - // with 75% segwit discount applied to the script size. - nSize += (32 + 4 + 1 + (107 / WITNESS_SCALE_FACTOR) + 4); - } else { - nSize += (32 + 4 + 1 + 107 + 4); // the 148 mentioned above - } - - return 3 * minRelayTxFee.GetFee(nSize); - } - - bool IsDust(const CFeeRate &minRelayTxFee) const - { - return (nValue < GetDustThreshold(minRelayTxFee)); - } - friend bool operator==(const CTxOut& a, const CTxOut& b) { return (a.nValue == b.nValue && diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index 4b34e73eb7..f8a99506c1 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -181,7 +181,5 @@ void BanTableModel::sort(int column, Qt::SortOrder order) bool BanTableModel::shouldShow() { - if (priv->size() > 0) - return true; - return false; + return priv->size() > 0; } diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index be79a67990..5c26baef9e 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -478,6 +478,7 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) connect(_clientModel, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int))); connect(_clientModel, SIGNAL(networkActiveChanged(bool)), this, SLOT(setNetworkActive(bool))); + modalOverlay->setKnownBestHeight(_clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(_clientModel->getHeaderTipTime())); setNumBlocks(_clientModel->getNumBlocks(), _clientModel->getLastBlockDate(), _clientModel->getVerificationProgress(NULL), false); connect(_clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool))); @@ -505,8 +506,6 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel) // initialize the disable state of the tray icon with the current value in the model. setTrayIconVisible(optionsModel->getHideTrayIcon()); } - - modalOverlay->setKnownBestHeight(clientModel->getHeaderTipHeight(), QDateTime::fromTime_t(clientModel->getHeaderTipTime())); } else { // Disable possibility to show main window via action toggleHideAction->setEnabled(false); diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index bb10e49422..de00eacdb9 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -9,6 +9,7 @@ #include "guiutil.h" #include "peertablemodel.h" +#include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "clientversion.h" @@ -36,6 +37,8 @@ ClientModel::ClientModel(OptionsModel *_optionsModel, QObject *parent) : banTableModel(0), pollTimer(0) { + cachedBestHeaderHeight = -1; + cachedBestHeaderTime = -1; peerTableModel = new PeerTableModel(this); banTableModel = new BanTableModel(this); pollTimer = new QTimer(this); @@ -74,18 +77,28 @@ int ClientModel::getNumBlocks() const int ClientModel::getHeaderTipHeight() const { - LOCK(cs_main); - if (!pindexBestHeader) - return 0; - return pindexBestHeader->nHeight; + if (cachedBestHeaderHeight == -1) { + // make sure we initially populate the cache via a cs_main lock + // otherwise we need to wait for a tip update + LOCK(cs_main); + if (pindexBestHeader) { + cachedBestHeaderHeight = pindexBestHeader->nHeight; + cachedBestHeaderTime = pindexBestHeader->GetBlockTime(); + } + } + return cachedBestHeaderHeight; } int64_t ClientModel::getHeaderTipTime() const { - LOCK(cs_main); - if (!pindexBestHeader) - return 0; - return pindexBestHeader->GetBlockTime(); + if (cachedBestHeaderTime == -1) { + LOCK(cs_main); + if (pindexBestHeader) { + cachedBestHeaderHeight = pindexBestHeader->nHeight; + cachedBestHeaderTime = pindexBestHeader->GetBlockTime(); + } + } + return cachedBestHeaderTime; } quint64 ClientModel::getTotalBytesRecv() const @@ -283,6 +296,11 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, const CB int64_t& nLastUpdateNotification = fHeader ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; + if (fHeader) { + // cache best headers time and height to reduce future cs_main locks + clientmodel->cachedBestHeaderHeight = pIndex->nHeight; + clientmodel->cachedBestHeaderTime = pIndex->GetBlockTime(); + } // if we are in-sync, update the UI regardless of last update time if (!initialSync || now - nLastUpdateNotification > MODEL_UPDATE_DELAY) { //pass a async signal to the UI thread diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 2c10e633b8..4c92e2144e 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -8,6 +8,8 @@ #include <QObject> #include <QDateTime> +#include <atomic> + class AddressTableModel; class BanTableModel; class OptionsModel; @@ -81,6 +83,10 @@ public: QString formatClientStartupTime() const; QString dataDir() const; + // caches for the best header + mutable std::atomic<int> cachedBestHeaderHeight; + mutable std::atomic<int64_t> cachedBestHeaderTime; + private: OptionsModel *optionsModel; PeerTableModel *peerTableModel; diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 1d19c65753..135cf6f701 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -15,6 +15,7 @@ #include "wallet/coincontrol.h" #include "init.h" +#include "policy/fees.h" #include "policy/policy.h" #include "validation.h" // For mempool #include "wallet/wallet.h" @@ -433,8 +434,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) { CTxOut txout(amount, (CScript)std::vector<unsigned char>(24, 0)); txDummy.vout.push_back(txout); - if (txout.IsDust(dustRelayFee)) - fDust = true; + fDust |= IsDust(txout, ::dustRelayFee); } } @@ -512,9 +512,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) nBytes -= 34; // Fee - nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, mempool); - if (nPayFee > 0 && coinControl->nMinimumTotalFee > nPayFee) - nPayFee = coinControl->nMinimumTotalFee; + nPayFee = CWallet::GetMinimumFee(nBytes, nTxConfirmTarget, ::mempool, ::feeEstimator); if (nPayAmount > 0) { @@ -526,10 +524,10 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (nChange > 0 && nChange < MIN_CHANGE) { CTxOut txout(nChange, (CScript)std::vector<unsigned char>(24, 0)); - if (txout.IsDust(dustRelayFee)) + if (IsDust(txout, ::dustRelayFee)) { if (CoinControlDialog::fSubtractFeeFromAmount) // dust-change will be raised until no dust - nChange = txout.GetDustThreshold(dustRelayFee); + nChange = GetDustThreshold(txout, ::dustRelayFee); else { nPayFee += nChange; @@ -573,7 +571,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) l5->setText(((nBytes > 0) ? ASYMP_UTF8 : "") + QString::number(nBytes)); // Bytes l7->setText(fDust ? tr("yes") : tr("no")); // Dust l8->setText(BitcoinUnits::formatWithUnit(nDisplayUnit, nChange)); // Change - if (nPayFee > 0 && (coinControl->nMinimumTotalFee < nPayFee)) + if (nPayFee > 0) { l3->setText(ASYMP_UTF8 + l3->text()); l4->setText(ASYMP_UTF8 + l4->text()); @@ -592,7 +590,7 @@ void CoinControlDialog::updateLabels(WalletModel *model, QDialog* dialog) if (payTxFee.GetFeePerK() > 0) dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), payTxFee.GetFeePerK()) / 1000; else { - dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), mempool.estimateSmartFee(nTxConfirmTarget).GetFeePerK()) / 1000; + dFeeVary = (double)std::max(CWallet::GetRequiredFee(1000), ::feeEstimator.estimateSmartFee(nTxConfirmTarget, NULL, ::mempool).GetFeePerK()) / 1000; } QString toolTip4 = tr("Can vary +/- %1 satoshi(s) per input.").arg(dFeeVary); diff --git a/src/qt/forms/sendcoinsdialog.ui b/src/qt/forms/sendcoinsdialog.ui index 52256ca5c4..89f9c25d14 100644 --- a/src/qt/forms/sendcoinsdialog.ui +++ b/src/qt/forms/sendcoinsdialog.ui @@ -862,19 +862,6 @@ </widget> </item> <item> - <widget class="QRadioButton" name="radioCustomAtLeast"> - <property name="toolTip"> - <string>If the custom fee is set to 1000 satoshis and the transaction is only 250 bytes, then "per kilobyte" only pays 250 satoshis in fee, while "total at least" pays 1000 satoshis. For transactions bigger than a kilobyte both pay by kilobyte.</string> - </property> - <property name="text"> - <string>total at least</string> - </property> - <attribute name="buttonGroup"> - <string notr="true">groupCustomFee</string> - </attribute> - </widget> - </item> - <item> <widget class="BitcoinAmountField" name="customFee"/> </item> <item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 3f3f9b9ccb..bffa81137b 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -251,7 +251,7 @@ bool isDust(const QString& address, const CAmount& amount) CTxDestination dest = CBitcoinAddress(address.toStdString()).Get(); CScript script = GetScriptForDestination(dest); CTxOut txOut(amount, script); - return txOut.IsDust(dustRelayFee); + return IsDust(txOut, ::dustRelayFee); } QString HtmlEscape(const QString& str, bool fMultiLine) diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index dd75f12076..c31a7a478d 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -219,14 +219,16 @@ void PaymentServer::ipcParseCommandLine(int argc, char* argv[]) if (GUIUtil::parseBitcoinURI(arg, &r) && !r.address.isEmpty()) { CBitcoinAddress address(r.address.toStdString()); + auto tempChainParams = CreateChainParams(CBaseChainParams::MAIN); - if (address.IsValid(Params(CBaseChainParams::MAIN))) + if (address.IsValid(*tempChainParams)) { SelectParams(CBaseChainParams::MAIN); } - else if (address.IsValid(Params(CBaseChainParams::TESTNET))) - { - SelectParams(CBaseChainParams::TESTNET); + else { + tempChainParams = CreateChainParams(CBaseChainParams::TESTNET); + if (address.IsValid(*tempChainParams)) + SelectParams(CBaseChainParams::TESTNET); } } } @@ -580,7 +582,7 @@ bool PaymentServer::processPaymentRequest(const PaymentRequestPlus& request, Sen // Extract and check amounts CTxOut txOut(sendingTo.second, sendingTo.first); - if (txOut.IsDust(dustRelayFee)) { + if (IsDust(txOut, ::dustRelayFee)) { Q_EMIT message(tr("Payment request error"), tr("Requested payment amount of %1 is too small (considered dust).") .arg(BitcoinUnits::formatWithUnit(optionsModel->getDisplayUnit(), sendingTo.second)), CClientUIInterface::MSG_ERROR); diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 974a71ddca..fff072fd4c 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -166,7 +166,7 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const switch(index.column()) { case NetNodeId: - return rec->nodeStats.nodeid; + return (qint64)rec->nodeStats.nodeid; case Address: return QString::fromStdString(rec->nodeStats.addrName); case Subversion: diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 5167232d6a..b200cb1127 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -98,7 +98,7 @@ class QtRPCTimerBase: public QObject, public RPCTimerBase { Q_OBJECT public: - QtRPCTimerBase(boost::function<void(void)>& _func, int64_t millis): + QtRPCTimerBase(std::function<void(void)>& _func, int64_t millis): func(_func) { timer.setSingleShot(true); @@ -110,7 +110,7 @@ private Q_SLOTS: void timeout() { func(); } private: QTimer timer; - boost::function<void(void)> func; + std::function<void(void)> func; }; class QtRPCTimerInterface: public RPCTimerInterface @@ -118,7 +118,7 @@ class QtRPCTimerInterface: public RPCTimerInterface public: ~QtRPCTimerInterface() {} const char *Name() { return "Qt"; } - RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) + RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) { return new QtRPCTimerBase(func, millis); } @@ -730,8 +730,14 @@ void RPCConsole::clear(bool clearHistory) ).arg(fixedFontInfo.family(), QString("%1pt").arg(consoleFontSize)) ); +#ifdef Q_OS_MAC + QString clsKey = "(⌘)-L"; +#else + QString clsKey = "Ctrl-L"; +#endif + message(CMD_REPLY, (tr("Welcome to the %1 RPC console.").arg(tr(PACKAGE_NAME)) + "<br>" + - tr("Use up and down arrows to navigate history, and <b>Ctrl-L</b> to clear screen.") + "<br>" + + tr("Use up and down arrows to navigate history, and %1 to clear screen.").arg("<b>"+clsKey+"</b>") + "<br>" + tr("Type <b>help</b> for an overview of available commands.")) + "<br><span class=\"secwarning\">" + tr("WARNING: Scammers have been active, telling users to type commands here, stealing their wallet contents. Do not use this console without fully understanding the ramification of a command.") + @@ -829,7 +835,7 @@ void RPCConsole::on_lineEdit_returnPressed() cmdBeforeBrowsing = QString(); - message(CMD_REQUEST, cmd); + message(CMD_REQUEST, QString::fromStdString(strFilteredCmd)); Q_EMIT cmdRequest(cmd); cmd = QString::fromStdString(strFilteredCmd); @@ -1118,7 +1124,7 @@ void RPCConsole::disconnectSelectedNode() for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data().toInt(); + NodeId id = nodes.at(i).data().toLongLong(); // Find the node, disconnect it and clear the selected node if(g_connman->DisconnectNode(id)) clearSelectedNode(); @@ -1135,7 +1141,7 @@ void RPCConsole::banSelectedNode(int bantime) for(int i = 0; i < nodes.count(); i++) { // Get currently selected peer address - NodeId id = nodes.at(i).data().toInt(); + NodeId id = nodes.at(i).data().toLongLong(); // Get currently selected peer address int detailNodeRow = clientModel->getPeerTableModel()->getRowByNodeId(id); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index ed7eab03f3..272ab9486a 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -21,6 +21,7 @@ #include "validation.h" // mempool and minRelayTxFee #include "ui_interface.h" #include "txmempool.h" +#include "policy/fees.h" #include "wallet/wallet.h" #include <QFontMetrics> @@ -30,8 +31,6 @@ #include <QTextDocument> #include <QTimer> -#define SEND_CONFIRM_DELAY 3 - SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : QDialog(parent), ui(new Ui::SendCoinsDialog), @@ -110,11 +109,9 @@ SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *p ui->groupFee->setId(ui->radioCustomFee, 1); ui->groupFee->button((int)std::max(0, std::min(1, settings.value("nFeeRadio").toInt())))->setChecked(true); ui->groupCustomFee->setId(ui->radioCustomPerKilobyte, 0); - ui->groupCustomFee->setId(ui->radioCustomAtLeast, 1); ui->groupCustomFee->button((int)std::max(0, std::min(1, settings.value("nCustomFeeRadio").toInt())))->setChecked(true); ui->customFee->setValue(settings.value("nTransactionFee").toLongLong()); ui->checkBoxMinimumFee->setChecked(settings.value("fPayOnlyMinFee").toBool()); - ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); minimizeFeeSection(settings.value("fFeeSectionMinimized").toBool()); } @@ -175,6 +172,9 @@ void SendCoinsDialog::setModel(WalletModel *_model) updateSmartFeeLabel(); updateGlobalFeeVariables(); + // set default rbf checkbox state + ui->optInRBF->setCheckState(model->getDefaultWalletRbf() ? Qt::Checked : Qt::Unchecked); + // set the smartfee-sliders default value (wallets default conf.target or last stored value) QSettings settings; if (settings.value("nSmartFeeSliderPosition").toInt() == 0) @@ -605,7 +605,6 @@ void SendCoinsDialog::updateFeeSectionControls() ui->checkBoxMinimumFee ->setEnabled(ui->radioCustomFee->isChecked()); ui->labelMinFeeWarning ->setEnabled(ui->radioCustomFee->isChecked()); ui->radioCustomPerKilobyte ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); - ui->radioCustomAtLeast ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked() && CoinControlDialog::coinControl->HasSelected()); ui->customFee ->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked()); } @@ -616,19 +615,12 @@ void SendCoinsDialog::updateGlobalFeeVariables() int nConfirmTarget = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; payTxFee = CFeeRate(0); - // set nMinimumTotalFee to 0 to not accidentally pay a custom fee - CoinControlDialog::coinControl->nMinimumTotalFee = 0; - // show the estimated required time for confirmation ui->confirmationTargetLabel->setText(GUIUtil::formatDurationStr(nConfirmTarget * Params().GetConsensus().nPowTargetSpacing) + " / " + tr("%n block(s)", "", nConfirmTarget)); } else { payTxFee = CFeeRate(ui->customFee->value()); - - // if user has selected to set a minimum absolute fee, pass the value to coincontrol - // set nMinimumTotalFee to 0 in case of user has selected that the fee is per KB - CoinControlDialog::coinControl->nMinimumTotalFee = ui->radioCustomAtLeast->isChecked() ? ui->customFee->value() : 0; } } @@ -660,7 +652,7 @@ void SendCoinsDialog::updateSmartFeeLabel() int nBlocksToConfirm = ui->sliderSmartFee->maximum() - ui->sliderSmartFee->value() + 2; int estimateFoundAtBlocks = nBlocksToConfirm; - CFeeRate feeRate = mempool.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocksToConfirm, &estimateFoundAtBlocks, ::mempool); if (feeRate <= CFeeRate(0)) // not enough data => minfee { ui->labelSmartFee->setText(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), @@ -827,21 +819,6 @@ void SendCoinsDialog::coinControlUpdateLabels() if (!model || !model->getOptionsModel()) return; - if (model->getOptionsModel()->getCoinControlFeatures()) - { - // enable minimum absolute fee UI controls - ui->radioCustomAtLeast->setVisible(true); - - // only enable the feature if inputs are selected - ui->radioCustomAtLeast->setEnabled(ui->radioCustomFee->isChecked() && !ui->checkBoxMinimumFee->isChecked() &&CoinControlDialog::coinControl->HasSelected()); - } - else - { - // in case coin control is disabled (=default), hide minimum absolute fee UI controls - ui->radioCustomAtLeast->setVisible(false); - return; - } - // set pay amounts CoinControlDialog::payAmounts.clear(); CoinControlDialog::fSubtractFeeFromAmount = false; diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index a402edc961..a932f129be 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -100,13 +100,14 @@ Q_SIGNALS: }; +#define SEND_CONFIRM_DELAY 3 class SendConfirmationDialog : public QMessageBox { Q_OBJECT public: - SendConfirmationDialog(const QString &title, const QString &text, int secDelay = 0, QWidget *parent = 0); + SendConfirmationDialog(const QString &title, const QString &text, int secDelay = SEND_CONFIRM_DELAY, QWidget *parent = 0); int exec(); private Q_SLOTS: diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index f794b6b382..ff1eb59f16 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -8,26 +8,46 @@ #include "qt/sendcoinsdialog.h" #include "qt/sendcoinsentry.h" #include "qt/transactiontablemodel.h" +#include "qt/transactionview.h" #include "qt/walletmodel.h" #include "test/test_bitcoin.h" #include "validation.h" #include "wallet/wallet.h" #include <QAbstractButton> +#include <QAction> #include <QApplication> +#include <QCheckBox> +#include <QPushButton> #include <QTimer> #include <QVBoxLayout> namespace { -//! Press "Yes" button in modal send confirmation dialog. -void ConfirmSend() +//! Press "Ok" button in message box dialog. +void ConfirmMessage(QString* text = nullptr) { - QTimer::singleShot(0, makeCallback([](Callback* callback) { + QTimer::singleShot(0, makeCallback([text](Callback* callback) { + for (QWidget* widget : QApplication::topLevelWidgets()) { + if (widget->inherits("QMessageBox")) { + QMessageBox* messageBox = qobject_cast<QMessageBox*>(widget); + if (text) *text = messageBox->text(); + messageBox->defaultButton()->click(); + } + } + delete callback; + }), SLOT(call())); +} + +//! Press "Yes" or "Cancel" buttons in modal send confirmation dialog. +void ConfirmSend(QString* text = nullptr, bool cancel = false) +{ + QTimer::singleShot(0, makeCallback([text, cancel](Callback* callback) { for (QWidget* widget : QApplication::topLevelWidgets()) { if (widget->inherits("SendConfirmationDialog")) { SendConfirmationDialog* dialog = qobject_cast<SendConfirmationDialog*>(widget); - QAbstractButton* button = dialog->button(QMessageBox::Yes); + if (text) *text = dialog->text(); + QAbstractButton* button = dialog->button(cancel ? QMessageBox::Cancel : QMessageBox::Yes); button->setEnabled(true); button->click(); } @@ -37,12 +57,16 @@ void ConfirmSend() } //! Send coins to address and return txid. -uint256 SendCoins(CWallet& wallet, SendCoinsDialog& sendCoinsDialog, const CBitcoinAddress& address, CAmount amount) +uint256 SendCoins(CWallet& wallet, SendCoinsDialog& sendCoinsDialog, const CBitcoinAddress& address, CAmount amount, bool rbf) { QVBoxLayout* entries = sendCoinsDialog.findChild<QVBoxLayout*>("entries"); SendCoinsEntry* entry = qobject_cast<SendCoinsEntry*>(entries->itemAt(0)->widget()); entry->findChild<QValidatedLineEdit*>("payTo")->setText(QString::fromStdString(address.ToString())); entry->findChild<BitcoinAmountField*>("payAmount")->setValue(amount); + sendCoinsDialog.findChild<QFrame*>("frameFee") + ->findChild<QFrame*>("frameFeeSelection") + ->findChild<QCheckBox*>("optInRBF") + ->setCheckState(rbf ? Qt::Checked : Qt::Unchecked); uint256 txid; boost::signals2::scoped_connection c(wallet.NotifyTransactionChanged.connect([&txid](CWallet*, const uint256& hash, ChangeType status) { if (status == CT_NEW) txid = hash; @@ -65,6 +89,42 @@ QModelIndex FindTx(const QAbstractItemModel& model, const uint256& txid) } return {}; } + +//! Request context menu (call method that is public in qt5, but protected in qt4). +void RequestContextMenu(QWidget* widget) +{ + class Qt4Hack : public QWidget + { + public: + using QWidget::customContextMenuRequested; + }; + static_cast<Qt4Hack*>(widget)->customContextMenuRequested({}); +} + +//! Invoke bumpfee on txid and check results. +void BumpFee(TransactionView& view, const uint256& txid, bool expectDisabled, std::string expectError, bool cancel) +{ + QTableView* table = view.findChild<QTableView*>("transactionView"); + QModelIndex index = FindTx(*table->selectionModel()->model(), txid); + QVERIFY2(index.isValid(), "Could not find BumpFee txid"); + + // Select row in table, invoke context menu, and make sure bumpfee action is + // enabled or disabled as expected. + QAction* action = view.findChild<QAction*>("bumpFeeAction"); + table->selectionModel()->select(index, QItemSelectionModel::ClearAndSelect | QItemSelectionModel::Rows); + action->setEnabled(expectDisabled); + RequestContextMenu(table); + QCOMPARE(action->isEnabled(), !expectDisabled); + + action->setEnabled(true); + QString text; + if (expectError.empty()) { + ConfirmSend(&text, cancel); + } else { + ConfirmMessage(&text); + } + action->trigger(); + QVERIFY(text.indexOf(QString::fromStdString(expectError)) != -1); } //! Simple qt wallet tests. @@ -80,13 +140,16 @@ QModelIndex FindTx(const QAbstractItemModel& model, const uint256& txid) // src/qt/test/test_bitcoin-qt -platform xcb # Linux // src/qt/test/test_bitcoin-qt -platform windows # Windows // src/qt/test/test_bitcoin-qt -platform cocoa # macOS -void WalletTests::walletTests() +void TestSendCoins() { - // Set up wallet and chain with 101 blocks (1 mature block for spending). + // Set up wallet and chain with 105 blocks (5 mature blocks for spending). TestChain100Setup test; - test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); + for (int i = 0; i < 5; ++i) { + test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); + } bitdb.MakeMock(); - CWallet wallet("wallet_test.dat"); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); + CWallet wallet(std::move(dbw)); bool firstRun; wallet.LoadWallet(firstRun); { @@ -100,19 +163,34 @@ void WalletTests::walletTests() // Create widgets for sending coins and listing transactions. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); SendCoinsDialog sendCoinsDialog(platformStyle.get()); + TransactionView transactionView(platformStyle.get()); OptionsModel optionsModel; WalletModel walletModel(platformStyle.get(), &wallet, &optionsModel); sendCoinsDialog.setModel(&walletModel); + transactionView.setModel(&walletModel); // Send two transactions, and verify they are added to transaction list. TransactionTableModel* transactionTableModel = walletModel.getTransactionTableModel(); - QCOMPARE(transactionTableModel->rowCount({}), 101); - uint256 txid1 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 5 * COIN); - uint256 txid2 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 10 * COIN); - QCOMPARE(transactionTableModel->rowCount({}), 103); + QCOMPARE(transactionTableModel->rowCount({}), 105); + uint256 txid1 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 5 * COIN, false /* rbf */); + uint256 txid2 = SendCoins(wallet, sendCoinsDialog, CBitcoinAddress(CKeyID()), 10 * COIN, true /* rbf */); + QCOMPARE(transactionTableModel->rowCount({}), 107); QVERIFY(FindTx(*transactionTableModel, txid1).isValid()); QVERIFY(FindTx(*transactionTableModel, txid2).isValid()); + // Call bumpfee. Test disabled, canceled, enabled, then failing cases. + BumpFee(transactionView, txid1, true /* expect disabled */, "not BIP 125 replaceable" /* expected error */, false /* cancel */); + BumpFee(transactionView, txid2, false /* expect disabled */, {} /* expected error */, true /* cancel */); + BumpFee(transactionView, txid2, false /* expect disabled */, {} /* expected error */, false /* cancel */); + BumpFee(transactionView, txid2, true /* expect disabled */, "already bumped" /* expected error */, false /* cancel */); + bitdb.Flush(true); bitdb.Reset(); } + +} + +void WalletTests::walletTests() +{ + TestSendCoins(); +} diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index d81188895b..233fc08772 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -293,13 +293,12 @@ QString TransactionDesc::toHTML(CWallet *wallet, CWalletTx &wtx, TransactionReco { COutPoint prevout = txin.prevout; - CCoins prev; - if(pcoinsTip->GetCoins(prevout.hash, prev)) + Coin prev; + if(pcoinsTip->GetCoin(prevout, prev)) { - if (prevout.n < prev.vout.size()) { strHTML += "<li>"; - const CTxOut &vout = prev.vout[prevout.n]; + const CTxOut &vout = prev.out; CTxDestination address; if (ExtractDestination(vout.scriptPubKey, address)) { diff --git a/src/qt/transactionrecord.cpp b/src/qt/transactionrecord.cpp index a9d9b6887e..0090b0c74b 100644 --- a/src/qt/transactionrecord.cpp +++ b/src/qt/transactionrecord.cpp @@ -18,14 +18,8 @@ */ bool TransactionRecord::showTransaction(const CWalletTx &wtx) { - if (wtx.IsCoinBase()) - { - // Ensures we show generated coins / mined transactions at depth 1 - if (!wtx.IsInMainChain()) - { - return false; - } - } + // There are currently no cases where we hide transactions, but + // we may want to use this in the future for things like RBF. return true; } @@ -252,13 +246,13 @@ void TransactionRecord::updateStatus(const CWalletTx &wtx) status.status = TransactionStatus::Confirmed; } } - + status.needsUpdate = false; } bool TransactionRecord::statusUpdateNeeded() { AssertLockHeld(cs_main); - return status.cur_num_blocks != chainActive.Height(); + return status.cur_num_blocks != chainActive.Height() || status.needsUpdate; } QString TransactionRecord::getTxID() const diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index 5aabbbffa8..59f681224f 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -61,6 +61,8 @@ public: /** Current number of blocks (to know whether cached status is still valid) */ int cur_num_blocks; + + bool needsUpdate; }; /** UI model for a transaction. A core transaction can be represented by multiple UI transactions if it has diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 61466c8ed1..f27abc2104 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -168,6 +168,10 @@ public: case CT_UPDATED: // Miscellaneous updates -- nothing to do, status update will take care of this, and is only computed for // visible transactions. + for (int i = lowerIndex; i < upperIndex; i++) { + TransactionRecord *rec = &cachedWallet[i]; + rec->status.needsUpdate = true; + } break; } } diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 30f4db9450..e3e070b27f 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -11,6 +11,7 @@ #include "guiutil.h" #include "optionsmodel.h" #include "platformstyle.h" +#include "sendcoinsdialog.h" #include "transactiondescdialog.h" #include "transactionfilterproxy.h" #include "transactionrecord.h" @@ -37,7 +38,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *parent) : QWidget(parent), model(0), transactionProxyModel(0), - transactionView(0), abandonAction(0), columnResizingFixer(0) + transactionView(0), abandonAction(0), bumpFeeAction(0), columnResizingFixer(0) { // Build filter row setContentsMargins(0,0,0,0); @@ -135,9 +136,12 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa view->installEventFilter(this); transactionView = view; + transactionView->setObjectName("transactionView"); // Actions abandonAction = new QAction(tr("Abandon transaction"), this); + bumpFeeAction = new QAction(tr("Increase transaction fee"), this); + bumpFeeAction->setObjectName("bumpFeeAction"); QAction *copyAddressAction = new QAction(tr("Copy address"), this); QAction *copyLabelAction = new QAction(tr("Copy label"), this); QAction *copyAmountAction = new QAction(tr("Copy amount"), this); @@ -148,6 +152,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa QAction *showDetailsAction = new QAction(tr("Show transaction details"), this); contextMenu = new QMenu(this); + contextMenu->setObjectName("contextMenu"); contextMenu->addAction(copyAddressAction); contextMenu->addAction(copyLabelAction); contextMenu->addAction(copyAmountAction); @@ -156,6 +161,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa contextMenu->addAction(copyTxPlainText); contextMenu->addAction(showDetailsAction); contextMenu->addSeparator(); + contextMenu->addAction(bumpFeeAction); contextMenu->addAction(abandonAction); contextMenu->addAction(editLabelAction); @@ -173,6 +179,7 @@ TransactionView::TransactionView(const PlatformStyle *platformStyle, QWidget *pa connect(view, SIGNAL(doubleClicked(QModelIndex)), this, SIGNAL(doubleClicked(QModelIndex))); connect(view, SIGNAL(customContextMenuRequested(QPoint)), this, SLOT(contextualMenu(QPoint))); + connect(bumpFeeAction, SIGNAL(triggered()), this, SLOT(bumpFee())); connect(abandonAction, SIGNAL(triggered()), this, SLOT(abandonTx())); connect(copyAddressAction, SIGNAL(triggered()), this, SLOT(copyAddress())); connect(copyLabelAction, SIGNAL(triggered()), this, SLOT(copyLabel())); @@ -372,10 +379,11 @@ void TransactionView::contextualMenu(const QPoint &point) uint256 hash; hash.SetHex(selection.at(0).data(TransactionTableModel::TxHashRole).toString().toStdString()); abandonAction->setEnabled(model->transactionCanBeAbandoned(hash)); + bumpFeeAction->setEnabled(model->transactionCanBeBumped(hash)); if(index.isValid()) { - contextMenu->exec(QCursor::pos()); + contextMenu->popup(transactionView->viewport()->mapToGlobal(point)); } } @@ -397,6 +405,24 @@ void TransactionView::abandonTx() model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, false); } +void TransactionView::bumpFee() +{ + if(!transactionView || !transactionView->selectionModel()) + return; + QModelIndexList selection = transactionView->selectionModel()->selectedRows(0); + + // get the hash from the TxHashRole (QVariant / QString) + uint256 hash; + QString hashQStr = selection.at(0).data(TransactionTableModel::TxHashRole).toString(); + hash.SetHex(hashQStr.toStdString()); + + // Bump tx fee over the walletModel + if (model->bumpFee(hash)) { + // Update the table + model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, true); + } +} + void TransactionView::copyAddress() { GUIUtil::copyEntryData(transactionView, 0, TransactionTableModel::AddressRole); diff --git a/src/qt/transactionview.h b/src/qt/transactionview.h index 595701cdd9..52e57cae4c 100644 --- a/src/qt/transactionview.h +++ b/src/qt/transactionview.h @@ -76,6 +76,7 @@ private: QDateTimeEdit *dateFrom; QDateTimeEdit *dateTo; QAction *abandonAction; + QAction *bumpFeeAction; QWidget *createDateRangeWidget(); @@ -99,6 +100,7 @@ private Q_SLOTS: void openThirdPartyTxUrl(QString url); void updateWatchOnlyColumn(bool fHaveWatchOnly); void abandonTx(); + void bumpFee(); Q_SIGNALS: void doubleClicked(const QModelIndex&); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index ebcac53c25..8df0e481bd 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -8,23 +8,29 @@ #include "consensus/validation.h" #include "guiconstants.h" #include "guiutil.h" +#include "optionsmodel.h" #include "paymentserver.h" #include "recentrequeststablemodel.h" +#include "sendcoinsdialog.h" #include "transactiontablemodel.h" #include "base58.h" +#include "chain.h" #include "keystore.h" #include "validation.h" #include "net.h" // for g_connman +#include "policy/rbf.h" #include "sync.h" #include "ui_interface.h" #include "util.h" // for GetBoolArg +#include "wallet/feebumper.h" #include "wallet/wallet.h" #include "wallet/walletdb.h" // for BackupWallet #include <stdint.h> #include <QDebug> +#include <QMessageBox> #include <QSet> #include <QTimer> @@ -62,14 +68,7 @@ CAmount WalletModel::getBalance(const CCoinControl *coinControl) const { if (coinControl) { - CAmount nBalance = 0; - std::vector<COutput> vCoins; - wallet->AvailableCoins(vCoins, true, coinControl); - BOOST_FOREACH(const COutput& out, vCoins) - if(out.fSpendable) - nBalance += out.tx->tx->vout[out.i].nValue; - - return nBalance; + return wallet->GetAvailableBalance(coinControl); } return wallet->GetBalance(); @@ -594,38 +593,11 @@ bool WalletModel::isSpent(const COutPoint& outpoint) const // AvailableCoins + LockedCoins grouped by wallet address (put change in one group with wallet address) void WalletModel::listCoins(std::map<QString, std::vector<COutput> >& mapCoins) const { - std::vector<COutput> vCoins; - wallet->AvailableCoins(vCoins); - - LOCK2(cs_main, wallet->cs_wallet); // ListLockedCoins, mapWallet - std::vector<COutPoint> vLockedCoins; - wallet->ListLockedCoins(vLockedCoins); - - // add locked coins - BOOST_FOREACH(const COutPoint& outpoint, vLockedCoins) - { - if (!wallet->mapWallet.count(outpoint.hash)) continue; - int nDepth = wallet->mapWallet[outpoint.hash].GetDepthInMainChain(); - if (nDepth < 0) continue; - COutput out(&wallet->mapWallet[outpoint.hash], outpoint.n, nDepth, true /* spendable */, true /* solvable */, true /* safe */); - if (outpoint.n < out.tx->tx->vout.size() && wallet->IsMine(out.tx->tx->vout[outpoint.n]) == ISMINE_SPENDABLE) - vCoins.push_back(out); - } - - BOOST_FOREACH(const COutput& out, vCoins) - { - COutput cout = out; - - while (wallet->IsChange(cout.tx->tx->vout[cout.i]) && cout.tx->tx->vin.size() > 0 && wallet->IsMine(cout.tx->tx->vin[0])) - { - if (!wallet->mapWallet.count(cout.tx->tx->vin[0].prevout.hash)) break; - cout = COutput(&wallet->mapWallet[cout.tx->tx->vin[0].prevout.hash], cout.tx->tx->vin[0].prevout.n, 0 /* depth */, true /* spendable */, true /* solvable */, true /* safe */); + for (auto& group : wallet->ListCoins()) { + auto& resultGroup = mapCoins[QString::fromStdString(CBitcoinAddress(group.first).ToString())]; + for (auto& coin : group.second) { + resultGroup.emplace_back(std::move(coin)); } - - CTxDestination address; - if(!out.fSpendable || !ExtractDestination(cout.tx->tx->vout[cout.i].scriptPubKey, address)) - continue; - mapCoins[QString::fromStdString(CBitcoinAddress(address).ToString())].push_back(out); } } @@ -655,11 +627,7 @@ void WalletModel::listLockedCoins(std::vector<COutPoint>& vOutpts) void WalletModel::loadReceiveRequests(std::vector<std::string>& vReceiveRequests) { - LOCK(wallet->cs_wallet); - BOOST_FOREACH(const PAIRTYPE(CTxDestination, CAddressBookData)& item, wallet->mapAddressBook) - BOOST_FOREACH(const PAIRTYPE(std::string, std::string)& item2, item.second.destdata) - if (item2.first.size() > 2 && item2.first.substr(0,2) == "rr") // receive request - vReceiveRequests.push_back(item2.second); + vReceiveRequests = wallet->GetDestValues("rr"); // receive request } bool WalletModel::saveReceiveRequest(const std::string &sAddress, const int64_t nId, const std::string &sRequest) @@ -679,11 +647,7 @@ bool WalletModel::saveReceiveRequest(const std::string &sAddress, const int64_t bool WalletModel::transactionCanBeAbandoned(uint256 hash) const { - LOCK2(cs_main, wallet->cs_wallet); - const CWalletTx *wtx = wallet->GetWalletTx(hash); - if (!wtx || wtx->isAbandoned() || wtx->GetDepthInMainChain() > 0 || wtx->InMempool()) - return false; - return true; + return wallet->TransactionCanBeAbandoned(hash); } bool WalletModel::abandonTransaction(uint256 hash) const @@ -692,6 +656,84 @@ bool WalletModel::abandonTransaction(uint256 hash) const return wallet->AbandonTransaction(hash); } +bool WalletModel::transactionCanBeBumped(uint256 hash) const +{ + LOCK2(cs_main, wallet->cs_wallet); + const CWalletTx *wtx = wallet->GetWalletTx(hash); + return wtx && SignalsOptInRBF(*wtx) && !wtx->mapValue.count("replaced_by_txid"); +} + +bool WalletModel::bumpFee(uint256 hash) +{ + std::unique_ptr<CFeeBumper> feeBump; + { + LOCK2(cs_main, wallet->cs_wallet); + feeBump.reset(new CFeeBumper(wallet, hash, nTxConfirmTarget, false, 0, true)); + } + if (feeBump->getResult() != BumpFeeResult::OK) + { + QMessageBox::critical(0, tr("Fee bump error"), tr("Increasing transaction fee failed") + "<br />(" + + (feeBump->getErrors().size() ? QString::fromStdString(feeBump->getErrors()[0]) : "") +")"); + return false; + } + + // allow a user based fee verification + QString questionString = tr("Do you want to increase the fee?"); + questionString.append("<br />"); + CAmount oldFee = feeBump->getOldFee(); + CAmount newFee = feeBump->getNewFee(); + questionString.append("<table style=\"text-align: left;\">"); + questionString.append("<tr><td>"); + questionString.append(tr("Current fee:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), oldFee)); + questionString.append("</td></tr><tr><td>"); + questionString.append(tr("Increase:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee - oldFee)); + questionString.append("</td></tr><tr><td>"); + questionString.append(tr("New fee:")); + questionString.append("</td><td>"); + questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), newFee)); + questionString.append("</td></tr></table>"); + SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); + confirmationDialog.exec(); + QMessageBox::StandardButton retval = (QMessageBox::StandardButton)confirmationDialog.result(); + + // cancel sign&broadcast if users doesn't want to bump the fee + if (retval != QMessageBox::Yes) { + return false; + } + + WalletModel::UnlockContext ctx(requestUnlock()); + if(!ctx.isValid()) + { + return false; + } + + // sign bumped transaction + bool res = false; + { + LOCK2(cs_main, wallet->cs_wallet); + res = feeBump->signTransaction(wallet); + } + if (!res) { + QMessageBox::critical(0, tr("Fee bump error"), tr("Can't sign transaction.")); + return false; + } + // commit the bumped transaction + { + LOCK2(cs_main, wallet->cs_wallet); + res = feeBump->commit(wallet); + } + if(!res) { + QMessageBox::critical(0, tr("Fee bump error"), tr("Could not commit transaction") + "<br />(" + + QString::fromStdString(feeBump->getErrors()[0])+")"); + return false; + } + return true; +} + bool WalletModel::isWalletEnabled() { return !GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET); diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 78e45dc369..16b0caed4e 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -207,6 +207,9 @@ public: bool transactionCanBeAbandoned(uint256 hash) const; bool abandonTransaction(uint256 hash) const; + bool transactionCanBeBumped(uint256 hash) const; + bool bumpFee(uint256 hash); + static bool isWalletEnabled(); bool hdEnabled() const; diff --git a/src/random.cpp b/src/random.cpp index 6bcd0a70ba..de7553c825 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -16,6 +16,8 @@ #include <stdlib.h> #include <limits> +#include <chrono> +#include <thread> #ifndef WIN32 #include <sys/time.h> @@ -32,6 +34,8 @@ #include <sys/sysctl.h> #endif +#include <mutex> + #include <openssl/err.h> #include <openssl/rand.h> @@ -43,15 +47,22 @@ static void RandFailure() static inline int64_t GetPerformanceCounter() { - int64_t nCounter = 0; -#ifdef WIN32 - QueryPerformanceCounter((LARGE_INTEGER*)&nCounter); + // Read the hardware time stamp counter when available. + // See https://en.wikipedia.org/wiki/Time_Stamp_Counter for more information. +#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) + return __rdtsc(); +#elif !defined(_MSC_VER) && defined(__i386__) + uint64_t r = 0; + __asm__ volatile ("rdtsc" : "=A"(r)); // Constrain the r variable to the eax:edx pair. + return r; +#elif !defined(_MSC_VER) && (defined(__x86_64__) || defined(__amd64__)) + uint64_t r1 = 0, r2 = 0; + __asm__ volatile ("rdtsc" : "=a"(r1), "=d"(r2)); // Constrain r1 to rax and r2 to rdx. + return (r2 << 32) | r1; #else - timeval t; - gettimeofday(&t, NULL); - nCounter = (int64_t)(t.tv_sec * 1000000 + t.tv_usec); + // Fall back to using C++11 clock (usually microsecond or nanosecond precision) + return std::chrono::high_resolution_clock::now().time_since_epoch().count(); #endif - return nCounter; } void RandAddSeed() @@ -192,6 +203,43 @@ void GetRandBytes(unsigned char* buf, int num) } } +static void AddDataToRng(void* data, size_t len); + +void RandAddSeedSleep() +{ + int64_t nPerfCounter1 = GetPerformanceCounter(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + int64_t nPerfCounter2 = GetPerformanceCounter(); + + // Combine with and update state + AddDataToRng(&nPerfCounter1, sizeof(nPerfCounter1)); + AddDataToRng(&nPerfCounter2, sizeof(nPerfCounter2)); + + memory_cleanse(&nPerfCounter1, sizeof(nPerfCounter1)); + memory_cleanse(&nPerfCounter2, sizeof(nPerfCounter2)); +} + + +static std::mutex cs_rng_state; +static unsigned char rng_state[32] = {0}; +static uint64_t rng_counter = 0; + +static void AddDataToRng(void* data, size_t len) { + CSHA512 hasher; + hasher.Write((const unsigned char*)&len, sizeof(len)); + hasher.Write((const unsigned char*)data, len); + unsigned char buf[64]; + { + std::unique_lock<std::mutex> lock(cs_rng_state); + hasher.Write(rng_state, sizeof(rng_state)); + hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); + ++rng_counter; + hasher.Finalize(buf); + memcpy(rng_state, buf + 32, 32); + } + memory_cleanse(buf, 64); +} + void GetStrongRandBytes(unsigned char* out, int num) { assert(num <= 32); @@ -207,8 +255,17 @@ void GetStrongRandBytes(unsigned char* out, int num) GetOSRand(buf); hasher.Write(buf, 32); + // Combine with and update state + { + std::unique_lock<std::mutex> lock(cs_rng_state); + hasher.Write(rng_state, sizeof(rng_state)); + hasher.Write((const unsigned char*)&rng_counter, sizeof(rng_counter)); + ++rng_counter; + hasher.Finalize(buf); + memcpy(rng_state, buf + 32, 32); + } + // Produce output - hasher.Finalize(buf); memcpy(out, buf, num); memory_cleanse(buf, 64); } @@ -240,26 +297,22 @@ uint256 GetRandHash() return hash; } -FastRandomContext::FastRandomContext(bool fDeterministic) +void FastRandomContext::RandomSeed() { - // The seed values have some unlikely fixed points which we avoid. - if (fDeterministic) { - Rz = Rw = 11; - } else { - uint32_t tmp; - do { - GetRandBytes((unsigned char*)&tmp, 4); - } while (tmp == 0 || tmp == 0x9068ffffU); - Rz = tmp; - do { - GetRandBytes((unsigned char*)&tmp, 4); - } while (tmp == 0 || tmp == 0x464fffffU); - Rw = tmp; - } + uint256 seed = GetRandHash(); + rng.SetKey(seed.begin(), 32); + requires_seed = false; +} + +FastRandomContext::FastRandomContext(const uint256& seed) : requires_seed(false), bytebuf_size(0), bitbuf_size(0) +{ + rng.SetKey(seed.begin(), 32); } bool Random_SanityCheck() { + uint64_t start = GetPerformanceCounter(); + /* This does not measure the quality of randomness, but it does test that * OSRandom() overwrites all 32 bytes of the output given a maximum * number of tries. @@ -286,5 +339,25 @@ bool Random_SanityCheck() tries += 1; } while (num_overwritten < NUM_OS_RANDOM_BYTES && tries < MAX_TRIES); - return (num_overwritten == NUM_OS_RANDOM_BYTES); /* If this failed, bailed out after too many tries */ + if (num_overwritten != NUM_OS_RANDOM_BYTES) return false; /* If this failed, bailed out after too many tries */ + + // Check that GetPerformanceCounter increases at least during a GetOSRand() call + 1ms sleep. + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + uint64_t stop = GetPerformanceCounter(); + if (stop == start) return false; + + // We called GetPerformanceCounter. Use it as entropy. + RAND_add((const unsigned char*)&start, sizeof(start), 1); + RAND_add((const unsigned char*)&stop, sizeof(stop), 1); + + return true; +} + +FastRandomContext::FastRandomContext(bool fDeterministic) : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) +{ + if (!fDeterministic) { + return; + } + uint256 seed; + rng.SetKey(seed.begin(), 32); } diff --git a/src/random.h b/src/random.h index 0464bdce14..6a63d57429 100644 --- a/src/random.h +++ b/src/random.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_RANDOM_H #define BITCOIN_RANDOM_H +#include "crypto/chacha20.h" +#include "crypto/common.h" #include "uint256.h" #include <stdint.h> @@ -22,6 +24,13 @@ int GetRandInt(int nMax); uint256 GetRandHash(); /** + * Add a little bit of randomness to the output of GetStrongRangBytes. + * This sleeps for a millisecond, so should only be called when there is + * no other work to be done. + */ +void RandAddSeedSleep(); + +/** * Function to gather random data from multiple sources, failing whenever any * of those source fail to provide a result. */ @@ -33,17 +42,79 @@ void GetStrongRandBytes(unsigned char* buf, int num); * This class is not thread-safe. */ class FastRandomContext { +private: + bool requires_seed; + ChaCha20 rng; + + unsigned char bytebuf[64]; + int bytebuf_size; + + uint64_t bitbuf; + int bitbuf_size; + + void RandomSeed(); + + void FillByteBuffer() + { + if (requires_seed) { + RandomSeed(); + } + rng.Output(bytebuf, sizeof(bytebuf)); + bytebuf_size = sizeof(bytebuf); + } + + void FillBitBuffer() + { + bitbuf = rand64(); + bitbuf_size = 64; + } + public: - explicit FastRandomContext(bool fDeterministic=false); + explicit FastRandomContext(bool fDeterministic = false); + + /** Initialize with explicit seed (only for testing) */ + explicit FastRandomContext(const uint256& seed); + + /** Generate a random 64-bit integer. */ + uint64_t rand64() + { + if (bytebuf_size < 8) FillByteBuffer(); + uint64_t ret = ReadLE64(bytebuf + 64 - bytebuf_size); + bytebuf_size -= 8; + return ret; + } + + /** Generate a random (bits)-bit integer. */ + uint64_t randbits(int bits) { + if (bits == 0) { + return 0; + } else if (bits > 32) { + return rand64() >> (64 - bits); + } else { + if (bitbuf_size < bits) FillBitBuffer(); + uint64_t ret = bitbuf & (~(uint64_t)0 >> (64 - bits)); + bitbuf >>= bits; + bitbuf_size -= bits; + return ret; + } + } - uint32_t rand32() { - Rz = 36969 * (Rz & 65535) + (Rz >> 16); - Rw = 18000 * (Rw & 65535) + (Rw >> 16); - return (Rw << 16) + Rz; + /** Generate a random integer in the range [0..range). */ + uint64_t randrange(uint64_t range) + { + --range; + int bits = CountBits(range); + while (true) { + uint64_t ret = randbits(bits); + if (ret <= range) return ret; + } } - uint32_t Rz; - uint32_t Rw; + /** Generate a random 32-bit integer. */ + uint32_t rand32() { return randbits(32); } + + /** Generate a random boolean. */ + bool randbool() { return randbits(1); } }; /* Number of random bytes returned by GetOSRand. diff --git a/src/rest.cpp b/src/rest.cpp index 9dcaf269d6..b08d7153b1 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -5,6 +5,7 @@ #include "chain.h" #include "chainparams.h" +#include "core_io.h" #include "primitives/block.h" #include "primitives/transaction.h" #include "validation.h" @@ -41,25 +42,24 @@ static const struct { }; struct CCoin { - uint32_t nTxVer; // Don't call this nVersion, that name has a special meaning inside IMPLEMENT_SERIALIZE uint32_t nHeight; CTxOut out; ADD_SERIALIZE_METHODS; + CCoin() : nHeight(0) {} + CCoin(Coin&& in) : nHeight(in.nHeight), out(std::move(in.out)) {} + template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nTxVer); + uint32_t nTxVerDummy = 0; + READWRITE(nTxVerDummy); READWRITE(nHeight); READWRITE(out); } }; -/* Defined in rawtransaction.cpp */ -void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); - static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string message) { req->WriteHeader("Content-Type", "text/plain"); @@ -383,7 +383,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) case RF_JSON: { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, hashBlock, objTx); + TxToUniv(*tx, hashBlock, objTx); std::string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -512,22 +512,11 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) view.SetBackend(viewMempool); // switch cache backend to db+mempool in case user likes to query mempool for (size_t i = 0; i < vOutPoints.size(); i++) { - CCoins coins; - uint256 hash = vOutPoints[i].hash; bool hit = false; - if (view.GetCoins(hash, coins)) { - mempool.pruneSpent(hash, coins); - if (coins.IsAvailable(vOutPoints[i].n)) { - hit = true; - // Safe to index into vout here because IsAvailable checked if it's off the end of the array, or if - // n is valid but points to an already spent output (IsNull). - CCoin coin; - coin.nTxVer = coins.nVersion; - coin.nHeight = coins.nHeight; - coin.out = coins.vout.at(vOutPoints[i].n); - assert(!coin.out.IsNull()); - outs.push_back(coin); - } + Coin coin; + if (view.GetCoin(vOutPoints[i], coin) && !mempool.isSpent(vOutPoints[i])) { + hit = true; + outs.emplace_back(std::move(coin)); } hits.push_back(hit); @@ -571,13 +560,12 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) UniValue utxos(UniValue::VARR); BOOST_FOREACH (const CCoin& coin, outs) { UniValue utxo(UniValue::VOBJ); - utxo.push_back(Pair("txvers", (int32_t)coin.nTxVer)); utxo.push_back(Pair("height", (int32_t)coin.nHeight)); utxo.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); // include the script in a json output UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); + ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); utxo.push_back(Pair("scriptPubKey", o)); utxos.push_back(utxo); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 01066d0eb2..96871ce1dc 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -12,6 +12,8 @@ #include "coins.h" #include "consensus/validation.h" #include "validation.h" +#include "core_io.h" +#include "policy/feerate.h" #include "policy/policy.h" #include "primitives/transaction.h" #include "rpc/server.h" @@ -42,7 +44,6 @@ static std::condition_variable cond_blockchange; static CUpdatedBlock latestblock; extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); double GetDifficulty(const CBlockIndex* blockindex) { @@ -123,7 +124,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool tx if(txDetails) { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, uint256(), objTx); + TxToUniv(*tx, uint256(), objTx); txs.push_back(objTx); } else @@ -186,7 +187,7 @@ void RPCNotifyBlockChange(bool ibd, const CBlockIndex * pindex) latestblock.hash = pindex->GetBlockHash(); latestblock.height = pindex->nHeight; } - cond_blockchange.notify_all(); + cond_blockchange.notify_all(); } UniValue waitfornewblock(const JSONRPCRequest& request) @@ -412,6 +413,7 @@ UniValue getrawmempool(const JSONRPCRequest& request) throw std::runtime_error( "getrawmempool ( verbose )\n" "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" + "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n" "\nArguments:\n" "1. verbose (boolean, optional, default=false) True for a json object, false for array of transaction ids\n" "\nResult: (for verbose = false):\n" @@ -686,13 +688,16 @@ UniValue getblock(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "getblock \"blockhash\" ( verbose )\n" - "\nIf verbose is false, returns a string that is serialized, hex-encoded data for block 'hash'.\n" - "If verbose is true, returns an Object with information about block <hash>.\n" + "getblock \"blockhash\" ( verbosity ) \n" + "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" + "If verbosity is 1, returns an Object with information about block <hash>.\n" + "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n" "\nArguments:\n" "1. \"blockhash\" (string, required) The block hash\n" - "2. verbose (boolean, optional, default=true) true for a json object, false for the hex encoded data\n" - "\nResult (for verbose = true):\n" + "2. verbosity (numeric, optional, default=1) 0 for hex encoded data, 1 for a json object, and 2 for json object with transaction data\n" + "\nResult (for verbosity = 0):\n" + "\"data\" (string) A string that is serialized, hex-encoded data for block 'hash'.\n" + "\nResult (for verbosity = 1):\n" "{\n" " \"hash\" : \"hash\", (string) the block hash (same as provided)\n" " \"confirmations\" : n, (numeric) The number of confirmations, or -1 if the block is not on the main chain\n" @@ -716,8 +721,14 @@ UniValue getblock(const JSONRPCRequest& request) " \"previousblockhash\" : \"hash\", (string) The hash of the previous block\n" " \"nextblockhash\" : \"hash\" (string) The hash of the next block\n" "}\n" - "\nResult (for verbose=false):\n" - "\"data\" (string) A string that is serialized, hex-encoded data for block 'hash'.\n" + "\nResult (for verbosity = 2):\n" + "{\n" + " ..., Same output as verbosity = 1.\n" + " \"tx\" : [ (array of Objects) The transactions in the format of the getrawtransaction RPC. Different from verbosity = 1 \"tx\" result.\n" + " ,...\n" + " ],\n" + " ,... Same output as verbosity = 1.\n" + "}\n" "\nExamples:\n" + HelpExampleCli("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") + HelpExampleRpc("getblock", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\"") @@ -728,9 +739,13 @@ UniValue getblock(const JSONRPCRequest& request) std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); - bool fVerbose = true; - if (request.params.size() > 1) - fVerbose = request.params[1].get_bool(); + int verbosity = 1; + if (request.params.size() > 1) { + if(request.params[1].isNum()) + verbosity = request.params[1].get_int(); + else + verbosity = request.params[1].get_bool() ? 1 : 0; + } if (mapBlockIndex.count(hash) == 0) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); @@ -749,7 +764,7 @@ UniValue getblock(const JSONRPCRequest& request) // block). throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); - if (!fVerbose) + if (verbosity <= 0) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); ssBlock << block; @@ -757,7 +772,7 @@ UniValue getblock(const JSONRPCRequest& request) return strHex; } - return blockToJSON(block, pblockindex); + return blockToJSON(block, pblockindex, verbosity >= 2); } struct CCoinsStats @@ -766,13 +781,29 @@ struct CCoinsStats uint256 hashBlock; uint64_t nTransactions; uint64_t nTransactionOutputs; - uint64_t nSerializedSize; uint256 hashSerialized; + uint64_t nDiskSize; CAmount nTotalAmount; - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nSerializedSize(0), nTotalAmount(0) {} + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nTotalAmount(0) {} }; +static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + assert(!outputs.empty()); + ss << hash; + ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase); + stats.nTransactions++; + for (const auto output : outputs) { + ss << VARINT(output.first + 1); + ss << *(const CScriptBase*)(&output.second.out.scriptPubKey); + ss << VARINT(output.second.out.nValue); + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.out.nValue; + } + ss << VARINT(0); +} + //! Calculate statistics about the unspent transaction output set static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) { @@ -785,32 +816,29 @@ static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) stats.nHeight = mapBlockIndex.find(stats.hashBlock)->second->nHeight; } ss << stats.hashBlock; - CAmount nTotalAmount = 0; + uint256 prevkey; + std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { boost::this_thread::interruption_point(); - uint256 key; - CCoins coins; - if (pcursor->GetKey(key) && pcursor->GetValue(coins)) { - stats.nTransactions++; - ss << key; - for (unsigned int i=0; i<coins.vout.size(); i++) { - const CTxOut &out = coins.vout[i]; - if (!out.IsNull()) { - stats.nTransactionOutputs++; - ss << VARINT(i+1); - ss << out; - nTotalAmount += out.nValue; - } + COutPoint key; + Coin coin; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + if (!outputs.empty() && key.hash != prevkey) { + ApplyStats(stats, ss, prevkey, outputs); + outputs.clear(); } - stats.nSerializedSize += 32 + pcursor->GetValueSize(); - ss << VARINT(0); + prevkey = key.hash; + outputs[key.n] = std::move(coin); } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } + if (!outputs.empty()) { + ApplyStats(stats, ss, prevkey, outputs); + } stats.hashSerialized = ss.GetHash(); - stats.nTotalAmount = nTotalAmount; + stats.nDiskSize = view->EstimateSize(); return true; } @@ -876,8 +904,8 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) " \"bestblock\": \"hex\", (string) the best block hash hex\n" " \"transactions\": n, (numeric) The number of transactions\n" " \"txouts\": n, (numeric) The number of output transactions\n" - " \"bytes_serialized\": n, (numeric) The serialized size\n" " \"hash_serialized\": \"hash\", (string) The serialized hash\n" + " \"disk_size\": n, (numeric) The estimated size of the chainstate on disk\n" " \"total_amount\": x.xxx (numeric) The total amount\n" "}\n" "\nExamples:\n" @@ -894,8 +922,8 @@ UniValue gettxoutsetinfo(const JSONRPCRequest& request) ret.push_back(Pair("bestblock", stats.hashBlock.GetHex())); ret.push_back(Pair("transactions", (int64_t)stats.nTransactions)); ret.push_back(Pair("txouts", (int64_t)stats.nTransactionOutputs)); - ret.push_back(Pair("bytes_serialized", (int64_t)stats.nSerializedSize)); - ret.push_back(Pair("hash_serialized", stats.hashSerialized.GetHex())); + ret.push_back(Pair("hash_serialized_2", stats.hashSerialized.GetHex())); + ret.push_back(Pair("disk_size", stats.nDiskSize)); ret.push_back(Pair("total_amount", ValueFromAmount(stats.nTotalAmount))); } else { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); @@ -948,37 +976,37 @@ UniValue gettxout(const JSONRPCRequest& request) std::string strHash = request.params[0].get_str(); uint256 hash(uint256S(strHash)); int n = request.params[1].get_int(); + COutPoint out(hash, n); bool fMempool = true; if (request.params.size() > 2) fMempool = request.params[2].get_bool(); - CCoins coins; + Coin coin; if (fMempool) { LOCK(mempool.cs); CCoinsViewMemPool view(pcoinsTip, mempool); - if (!view.GetCoins(hash, coins)) + if (!view.GetCoin(out, coin) || mempool.isSpent(out)) { // TODO: filtering spent coins should be done by the CCoinsViewMemPool return NullUniValue; - mempool.pruneSpent(hash, coins); // TODO: this should be done by the CCoinsViewMemPool + } } else { - if (!pcoinsTip->GetCoins(hash, coins)) + if (!pcoinsTip->GetCoin(out, coin)) { return NullUniValue; + } } - if (n<0 || (unsigned int)n>=coins.vout.size() || coins.vout[n].IsNull()) - return NullUniValue; BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); CBlockIndex *pindex = it->second; ret.push_back(Pair("bestblock", pindex->GetBlockHash().GetHex())); - if ((unsigned int)coins.nHeight == MEMPOOL_HEIGHT) + if (coin.nHeight == MEMPOOL_HEIGHT) { ret.push_back(Pair("confirmations", 0)); - else - ret.push_back(Pair("confirmations", pindex->nHeight - coins.nHeight + 1)); - ret.push_back(Pair("value", ValueFromAmount(coins.vout[n].nValue))); + } else { + ret.push_back(Pair("confirmations", (int64_t)(pindex->nHeight - coin.nHeight + 1))); + } + ret.push_back(Pair("value", ValueFromAmount(coin.out.nValue))); UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coins.vout[n].scriptPubKey, o, true); + ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); ret.push_back(Pair("scriptPubKey", o)); - ret.push_back(Pair("version", coins.nVersion)); - ret.push_back(Pair("coinbase", coins.fCoinBase)); + ret.push_back(Pair("coinbase", (bool)coin.fCoinBase)); return ret; } @@ -1059,6 +1087,17 @@ static UniValue BIP9SoftForkDesc(const Consensus::Params& consensusParams, Conse rv.push_back(Pair("startTime", consensusParams.vDeployments[id].nStartTime)); rv.push_back(Pair("timeout", consensusParams.vDeployments[id].nTimeout)); rv.push_back(Pair("since", VersionBitsTipStateSinceHeight(consensusParams, id))); + if (THRESHOLD_STARTED == thresholdState) + { + UniValue statsUV(UniValue::VOBJ); + BIP9Stats statsStruct = VersionBitsTipStatistics(consensusParams, id); + statsUV.push_back(Pair("period", statsStruct.period)); + statsUV.push_back(Pair("threshold", statsStruct.threshold)); + statsUV.push_back(Pair("elapsed", statsStruct.elapsed)); + statsUV.push_back(Pair("count", statsStruct.count)); + statsUV.push_back(Pair("possible", statsStruct.possible)); + rv.push_back(Pair("statistics", statsUV)); + } return rv; } @@ -1104,7 +1143,14 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" - " \"since\": xx (numeric) height of the first block to which the status applies\n" + " \"since\": xx, (numeric) height of the first block to which the status applies\n" + " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" + " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" + " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" + " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" + " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" + " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" + " }\n" " }\n" " }\n" "}\n" @@ -1292,7 +1338,7 @@ UniValue getmempoolinfo(const JSONRPCRequest& request) " \"bytes\": xxxxx, (numeric) Sum of all virtual transaction sizes as defined in BIP 141. Differs from actual serialized size because witness data is discounted\n" " \"usage\": xxxxx, (numeric) Total memory usage for the mempool\n" " \"maxmempool\": xxxxx, (numeric) Maximum memory usage for the mempool\n" - " \"mempoolminfee\": xxxxx (numeric) Minimum fee for tx to be accepted\n" + " \"mempoolminfee\": xxxxx (numeric) Minimum feerate (" + CURRENCY_UNIT + " per KB) for tx to be accepted\n" "}\n" "\nExamples:\n" + HelpExampleCli("getmempoolinfo", "") @@ -1415,13 +1461,79 @@ UniValue reconsiderblock(const JSONRPCRequest& request) return NullUniValue; } +UniValue getchaintxstats(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() > 2) + throw std::runtime_error( + "getchaintxstats ( nblocks blockhash )\n" + "\nCompute statistics about the total number and rate of transactions in the chain.\n" + "\nArguments:\n" + "1. nblocks (numeric, optional) Size of the window in number of blocks (default: one month).\n" + "2. \"blockhash\" (string, optional) The hash of the block that ends the window.\n" + "\nResult:\n" + "{\n" + " \"time\": xxxxx, (numeric) The timestamp for the statistics in UNIX format.\n" + " \"txcount\": xxxxx, (numeric) The total number of transactions in the chain up to that point.\n" + " \"txrate\": x.xx, (numeric) The average rate of transactions per second in the window.\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("getchaintxstats", "") + + HelpExampleRpc("getchaintxstats", "2016") + ); + + const CBlockIndex* pindex; + int blockcount = 30 * 24 * 60 * 60 / Params().GetConsensus().nPowTargetSpacing; // By default: 1 month + + if (request.params.size() > 0 && !request.params[0].isNull()) { + blockcount = request.params[0].get_int(); + } + + bool havehash = request.params.size() > 1 && !request.params[1].isNull(); + uint256 hash; + if (havehash) { + hash = uint256S(request.params[1].get_str()); + } + + { + LOCK(cs_main); + if (havehash) { + auto it = mapBlockIndex.find(hash); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + pindex = it->second; + if (!chainActive.Contains(pindex)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Block is not in main chain"); + } + } else { + pindex = chainActive.Tip(); + } + } + + if (blockcount < 1 || blockcount >= pindex->nHeight) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid block count: should be between 1 and the block's height"); + } + + const CBlockIndex* pindexPast = pindex->GetAncestor(pindex->nHeight - blockcount); + int nTimeDiff = pindex->GetMedianTimePast() - pindexPast->GetMedianTimePast(); + int nTxDiff = pindex->nChainTx - pindexPast->nChainTx; + + UniValue ret(UniValue::VOBJ); + ret.push_back(Pair("time", (int64_t)pindex->nTime)); + ret.push_back(Pair("txcount", (int64_t)pindex->nChainTx)); + ret.push_back(Pair("txrate", ((double)nTxDiff) / nTimeDiff)); + + return ret; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafe argNames // --------------------- ------------------------ ----------------------- ------ ---------- { "blockchain", "getblockchaininfo", &getblockchaininfo, true, {} }, + { "blockchain", "getchaintxstats", &getchaintxstats, true, {"nblocks", "blockhash"} }, { "blockchain", "getbestblockhash", &getbestblockhash, true, {} }, { "blockchain", "getblockcount", &getblockcount, true, {} }, - { "blockchain", "getblock", &getblock, true, {"blockhash","verbose"} }, + { "blockchain", "getblock", &getblock, true, {"blockhash","verbosity|verbose"} }, { "blockchain", "getblockhash", &getblockhash, true, {"height"} }, { "blockchain", "getblockheader", &getblockheader, true, {"blockhash","verbose"} }, { "blockchain", "getchaintips", &getchaintips, true, {} }, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 1f3c4e52ad..a3a692c14d 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -77,8 +77,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listunspent", 0, "minconf" }, { "listunspent", 1, "maxconf" }, { "listunspent", 2, "addresses" }, - { "getblock", 1, "verbose" }, + { "listunspent", 4, "query_options" }, + { "getblock", 1, "verbosity" }, { "getblockheader", 1, "verbose" }, + { "getchaintxstats", 0, "nblocks" }, { "gettransaction", 1, "include_watchonly" }, { "getrawtransaction", 1, "verbose" }, { "createrawtransaction", 0, "inputs" }, @@ -106,6 +108,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getrawmempool", 0, "verbose" }, { "estimatefee", 0, "nblocks" }, { "estimatesmartfee", 0, "nblocks" }, + { "estimatesmartfee", 1, "conservative" }, + { "estimaterawfee", 0, "nblocks" }, + { "estimaterawfee", 1, "threshold" }, + { "estimaterawfee", 2, "horizon" }, { "prioritisetransaction", 1, "fee_delta" }, { "setban", 2, "bantime" }, { "setban", 3, "absolute" }, @@ -115,6 +121,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "bumpfee", 1, "options" }, { "logging", 0, "include" }, { "logging", 1, "exclude" }, + { "disconnectnode", 1, "nodeid" }, // Echo with conversion (For testing only) { "echojson", 0, "arg0" }, { "echojson", 1, "arg1" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index d234bb69ae..d744269df1 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -15,6 +15,7 @@ #include "validation.h" #include "miner.h" #include "net.h" +#include "policy/fees.h" #include "pow.h" #include "rpc/blockchain.h" #include "rpc/server.h" @@ -796,6 +797,7 @@ UniValue estimatefee(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( "estimatefee nblocks\n" + "\nDEPRECATED. Please use estimatesmartfee for more intelligent estimates." "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks. Uses virtual transaction size of transaction\n" "as defined in BIP 141 (witness data is discounted).\n" @@ -818,7 +820,7 @@ UniValue estimatefee(const JSONRPCRequest& request) if (nBlocks < 1) nBlocks = 1; - CFeeRate feeRate = mempool.estimateFee(nBlocks); + CFeeRate feeRate = ::feeEstimator.estimateFee(nBlocks); if (feeRate == CFeeRate(0)) return -1.0; @@ -827,16 +829,19 @@ UniValue estimatefee(const JSONRPCRequest& request) UniValue estimatesmartfee(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) throw std::runtime_error( - "estimatesmartfee nblocks\n" - "\nWARNING: This interface is unstable and may disappear or change!\n" + "estimatesmartfee nblocks (conservative)\n" "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" "confirmation within nblocks blocks if possible and return the number of blocks\n" "for which the estimate is valid. Uses virtual transaction size as defined\n" "in BIP 141 (witness data is discounted).\n" "\nArguments:\n" - "1. nblocks (numeric)\n" + "1. nblocks (numeric)\n" + "2. conservative (bool, optional, default=true) Whether to return a more conservative estimate which\n" + " also satisfies a longer history. A conservative estimate potentially returns a higher\n" + " feerate and is more likely to be sufficient for the desired target, but is not as\n" + " responsive to short term drops in the prevailing fee market\n" "\nResult:\n" "{\n" " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" @@ -853,15 +858,102 @@ UniValue estimatesmartfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)); int nBlocks = request.params[0].get_int(); + bool conservative = true; + if (request.params.size() > 1 && !request.params[1].isNull()) { + RPCTypeCheckArgument(request.params[1], UniValue::VBOOL); + conservative = request.params[1].get_bool(); + } UniValue result(UniValue::VOBJ); int answerFound; - CFeeRate feeRate = mempool.estimateSmartFee(nBlocks, &answerFound); + CFeeRate feeRate = ::feeEstimator.estimateSmartFee(nBlocks, &answerFound, ::mempool, conservative); result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); result.push_back(Pair("blocks", answerFound)); return result; } +UniValue estimaterawfee(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 1|| request.params.size() > 3) + throw std::runtime_error( + "estimaterawfee nblocks (threshold horizon)\n" + "\nWARNING: This interface is unstable and may disappear or change!\n" + "\nWARNING: This is an advanced API call that is tightly coupled to the specific\n" + " implementation of fee estimation. The parameters it can be called with\n" + " and the results it returns will change if the internal implementation changes.\n" + "\nEstimates the approximate fee per kilobyte needed for a transaction to begin\n" + "confirmation within nblocks blocks if possible. Uses virtual transaction size as defined\n" + "in BIP 141 (witness data is discounted).\n" + "\nArguments:\n" + "1. nblocks (numeric)\n" + "2. threshold (numeric, optional) The proportion of transactions in a given feerate range that must have been\n" + " confirmed within nblocks in order to consider those feerates as high enough and proceed to check\n" + " lower buckets. Default: 0.95\n" + "3. horizon (numeric, optional) How long a history of estimates to consider. 0=short, 1=medium, 2=long.\n" + " Default: 1\n" + "\nResult:\n" + "{\n" + " \"feerate\" : x.x, (numeric) estimate fee-per-kilobyte (in BTC)\n" + " \"decay\" : x.x, (numeric) exponential decay (per block) for historical moving average of confirmation data\n" + " \"scale\" : x, (numeric) The resolution of confirmation targets at this time horizon\n" + " \"pass\" : { (json object) information about the lowest range of feerates to succeed in meeting the threshold\n" + " \"startrange\" : x.x, (numeric) start of feerate range\n" + " \"endrange\" : x.x, (numeric) end of feerate range\n" + " \"withintarget\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed within target\n" + " \"totalconfirmed\" : x.x, (numeric) number of txs over history horizon in the feerate range that were confirmed at any point\n" + " \"inmempool\" : x.x, (numeric) current number of txs in mempool in the feerate range unconfirmed for at least target blocks\n" + " \"leftmempool\" : x.x, (numeric) number of txs over history horizon in the feerate range that left mempool unconfirmed after target\n" + " }\n" + " \"fail\" : { ... } (json object) information about the highest range of feerates to fail to meet the threshold\n" + "}\n" + "\n" + "A negative feerate is returned if no answer can be given.\n" + "\nExample:\n" + + HelpExampleCli("estimaterawfee", "6 0.9 1") + ); + + RPCTypeCheck(request.params, boost::assign::list_of(UniValue::VNUM)(UniValue::VNUM)(UniValue::VNUM), true); + RPCTypeCheckArgument(request.params[0], UniValue::VNUM); + int nBlocks = request.params[0].get_int(); + double threshold = 0.95; + if (!request.params[1].isNull()) + threshold = request.params[1].get_real(); + FeeEstimateHorizon horizon = FeeEstimateHorizon::MED_HALFLIFE; + if (!request.params[2].isNull()) { + int horizonInt = request.params[2].get_int(); + if (horizonInt < 0 || horizonInt > 2) { + throw JSONRPCError(RPC_TYPE_ERROR, "Invalid horizon for fee estimates"); + } else { + horizon = (FeeEstimateHorizon)horizonInt; + } + } + UniValue result(UniValue::VOBJ); + CFeeRate feeRate; + EstimationResult buckets; + feeRate = ::feeEstimator.estimateRawFee(nBlocks, threshold, horizon, &buckets); + + result.push_back(Pair("feerate", feeRate == CFeeRate(0) ? -1.0 : ValueFromAmount(feeRate.GetFeePerK()))); + result.push_back(Pair("decay", buckets.decay)); + result.push_back(Pair("scale", (int)buckets.scale)); + UniValue passbucket(UniValue::VOBJ); + passbucket.push_back(Pair("startrange", round(buckets.pass.start))); + passbucket.push_back(Pair("endrange", round(buckets.pass.end))); + passbucket.push_back(Pair("withintarget", round(buckets.pass.withinTarget * 100.0) / 100.0)); + passbucket.push_back(Pair("totalconfirmed", round(buckets.pass.totalConfirmed * 100.0) / 100.0)); + passbucket.push_back(Pair("inmempool", round(buckets.pass.inMempool * 100.0) / 100.0)); + passbucket.push_back(Pair("leftmempool", round(buckets.pass.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("pass", passbucket)); + UniValue failbucket(UniValue::VOBJ); + failbucket.push_back(Pair("startrange", round(buckets.fail.start))); + failbucket.push_back(Pair("endrange", round(buckets.fail.end))); + failbucket.push_back(Pair("withintarget", round(buckets.fail.withinTarget * 100.0) / 100.0)); + failbucket.push_back(Pair("totalconfirmed", round(buckets.fail.totalConfirmed * 100.0) / 100.0)); + failbucket.push_back(Pair("inmempool", round(buckets.fail.inMempool * 100.0) / 100.0)); + failbucket.push_back(Pair("leftmempool", round(buckets.fail.leftMempool * 100.0) / 100.0)); + result.push_back(Pair("fail", failbucket)); + return result; +} + static const CRPCCommand commands[] = { // category name actor (function) okSafeMode // --------------------- ------------------------ ----------------------- ---------- @@ -875,7 +967,9 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, true, {"nblocks","address","maxtries"} }, { "util", "estimatefee", &estimatefee, true, {"nblocks"} }, - { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks"} }, + { "util", "estimatesmartfee", &estimatesmartfee, true, {"nblocks", "conservative"} }, + + { "hidden", "estimaterawfee", &estimaterawfee, true, {"nblocks", "threshold", "horizon"} }, }; void RegisterMiningRPCCommands(CRPCTable &t) diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 7b8aa572d7..1f973a0c18 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -4,6 +4,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "base58.h" +#include "chain.h" #include "clientversion.h" #include "init.h" #include "validation.h" diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 44c6e6d308..10bf99eb38 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -76,7 +76,8 @@ UniValue getpeerinfo(const JSONRPCRequest& request) " {\n" " \"id\": n, (numeric) Peer index\n" " \"addr\":\"host:port\", (string) The ip address and port of the peer\n" - " \"addrlocal\":\"ip:port\", (string) local address\n" + " \"addrbind\":\"ip:port\", (string) Bind address of the connection to the peer\n" + " \"addrlocal\":\"ip:port\", (string) Local address as reported by the peer\n" " \"services\":\"xxxxxxxxxxxxxxxx\", (string) The services offered\n" " \"relaytxes\":true|false, (boolean) Whether peer has asked us to relay transactions to it\n" " \"lastsend\": ttt, (numeric) The time in seconds since epoch (Jan 1 1970 GMT) of the last send\n" @@ -133,6 +134,8 @@ UniValue getpeerinfo(const JSONRPCRequest& request) obj.push_back(Pair("addr", stats.addrName)); if (!(stats.addrLocal.empty())) obj.push_back(Pair("addrlocal", stats.addrLocal)); + if (stats.addrBind.IsValid()) + obj.push_back(Pair("addrbind", stats.addrBind.ToString())); obj.push_back(Pair("services", strprintf("%016x", stats.nServices))); obj.push_back(Pair("relaytxes", stats.fRelayTxes)); obj.push_back(Pair("lastsend", stats.nLastSend)); @@ -234,23 +237,43 @@ UniValue addnode(const JSONRPCRequest& request) UniValue disconnectnode(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 1) + if (request.fHelp || request.params.size() == 0 || request.params.size() >= 3) throw std::runtime_error( - "disconnectnode \"node\" \n" - "\nImmediately disconnects from the specified node.\n" + "disconnectnode \"[address]\" [nodeid]\n" + "\nImmediately disconnects from the specified peer node.\n" + "\nStrictly one out of 'address' and 'nodeid' can be provided to identify the node.\n" + "\nTo disconnect by nodeid, either set 'address' to the empty string, or call using the named 'nodeid' argument only.\n" "\nArguments:\n" - "1. \"node\" (string, required) The node (see getpeerinfo for nodes)\n" + "1. \"address\" (string, optional) The IP address/port of the node\n" + "2. \"nodeid\" (number, optional) The node ID (see getpeerinfo for node IDs)\n" "\nExamples:\n" + HelpExampleCli("disconnectnode", "\"192.168.0.6:8333\"") + + HelpExampleCli("disconnectnode", "\"\" 1") + HelpExampleRpc("disconnectnode", "\"192.168.0.6:8333\"") + + HelpExampleRpc("disconnectnode", "\"\", 1") ); if(!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - bool ret = g_connman->DisconnectNode(request.params[0].get_str()); - if (!ret) + bool success; + const UniValue &address_arg = request.params[0]; + const UniValue &id_arg = request.params.size() < 2 ? NullUniValue : request.params[1]; + + if (!address_arg.isNull() && id_arg.isNull()) { + /* handle disconnect-by-address */ + success = g_connman->DisconnectNode(address_arg.get_str()); + } else if (!id_arg.isNull() && (address_arg.isNull() || (address_arg.isStr() && address_arg.get_str().empty()))) { + /* handle disconnect-by-id */ + NodeId nodeid = (NodeId) id_arg.get_int64(); + success = g_connman->DisconnectNode(nodeid); + } else { + throw JSONRPCError(RPC_INVALID_PARAMS, "Only one of address and nodeid should be provided."); + } + + if (!success) { throw JSONRPCError(RPC_CLIENT_NODE_NOT_CONNECTED, "Node not found in connected nodes"); + } return NullUniValue; } @@ -607,7 +630,7 @@ static const CRPCCommand commands[] = { "network", "ping", &ping, true, {} }, { "network", "getpeerinfo", &getpeerinfo, true, {} }, { "network", "addnode", &addnode, true, {"node","command"} }, - { "network", "disconnectnode", &disconnectnode, true, {"node"} }, + { "network", "disconnectnode", &disconnectnode, true, {"address", "nodeid"} }, { "network", "getaddednodeinfo", &getaddednodeinfo, true, {"node"} }, { "network", "getnettotals", &getnettotals, true, {} }, { "network", "getnetworkinfo", &getnetworkinfo, true, {} }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 717e9d75f3..e27c2a77c7 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -34,77 +34,15 @@ #include <univalue.h> -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex) -{ - txnouttype type; - std::vector<CTxDestination> addresses; - int nRequired; - - out.push_back(Pair("asm", ScriptToAsmStr(scriptPubKey))); - if (fIncludeHex) - out.push_back(Pair("hex", HexStr(scriptPubKey.begin(), scriptPubKey.end()))); - - if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired)) { - out.push_back(Pair("type", GetTxnOutputType(type))); - return; - } - - out.push_back(Pair("reqSigs", nRequired)); - out.push_back(Pair("type", GetTxnOutputType(type))); - - UniValue a(UniValue::VARR); - BOOST_FOREACH(const CTxDestination& addr, addresses) - a.push_back(CBitcoinAddress(addr).ToString()); - out.push_back(Pair("addresses", a)); -} void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { - entry.push_back(Pair("txid", tx.GetHash().GetHex())); - entry.push_back(Pair("hash", tx.GetWitnessHash().GetHex())); - entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); - entry.push_back(Pair("vsize", (int)::GetVirtualTransactionSize(tx))); - entry.push_back(Pair("version", tx.nVersion)); - entry.push_back(Pair("locktime", (int64_t)tx.nLockTime)); - - UniValue vin(UniValue::VARR); - for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxIn& txin = tx.vin[i]; - UniValue in(UniValue::VOBJ); - if (tx.IsCoinBase()) - in.push_back(Pair("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); - else { - in.push_back(Pair("txid", txin.prevout.hash.GetHex())); - in.push_back(Pair("vout", (int64_t)txin.prevout.n)); - UniValue o(UniValue::VOBJ); - o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true))); - o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); - in.push_back(Pair("scriptSig", o)); - } - if (tx.HasWitness()) { - UniValue txinwitness(UniValue::VARR); - for (unsigned int j = 0; j < tx.vin[i].scriptWitness.stack.size(); j++) { - std::vector<unsigned char> item = tx.vin[i].scriptWitness.stack[j]; - txinwitness.push_back(HexStr(item.begin(), item.end())); - } - in.push_back(Pair("txinwitness", txinwitness)); - } - in.push_back(Pair("sequence", (int64_t)txin.nSequence)); - vin.push_back(in); - } - entry.push_back(Pair("vin", vin)); - UniValue vout(UniValue::VARR); - for (unsigned int i = 0; i < tx.vout.size(); i++) { - const CTxOut& txout = tx.vout[i]; - UniValue out(UniValue::VOBJ); - out.push_back(Pair("value", ValueFromAmount(txout.nValue))); - out.push_back(Pair("n", (int64_t)i)); - UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(txout.scriptPubKey, o, true); - out.push_back(Pair("scriptPubKey", o)); - vout.push_back(out); - } - entry.push_back(Pair("vout", vout)); + // Call into TxToUniv() in bitcoin-common to decode the transaction hex. + // + // Blockchain contextual information (confirmations and blocktime) is not + // available to code in bitcoin-common, so we query them here and push the + // data into the returned UniValue. + TxToUniv(tx, uint256(), entry); if (!hashBlock.IsNull()) { entry.push_back(Pair("blockhash", hashBlock.GetHex())); @@ -281,9 +219,10 @@ UniValue gettxoutproof(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); pblockindex = mapBlockIndex[hashBlock]; } else { - CCoins coins; - if (pcoinsTip->GetCoins(oneTxid, coins) && coins.nHeight > 0 && coins.nHeight <= chainActive.Height()) - pblockindex = chainActive[coins.nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, oneTxid); + if (!coin.IsSpent() && coin.nHeight > 0 && coin.nHeight <= chainActive.Height()) { + pblockindex = chainActive[coin.nHeight]; + } } if (pblockindex == NULL) @@ -525,7 +464,7 @@ UniValue decoderawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); UniValue result(UniValue::VOBJ); - TxToJSON(CTransaction(std::move(mtx)), uint256(), result); + TxToUniv(CTransaction(std::move(mtx)), uint256(), result); return result; } @@ -565,7 +504,7 @@ UniValue decodescript(const JSONRPCRequest& request) } else { // Empty scripts are valid } - ScriptPubKeyToJSON(script, r, false); + ScriptPubKeyToUniv(script, r, false); UniValue type; type = find_value(r, "type"); @@ -585,6 +524,11 @@ static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std:: UniValue entry(UniValue::VOBJ); entry.push_back(Pair("txid", txin.prevout.hash.ToString())); entry.push_back(Pair("vout", (uint64_t)txin.prevout.n)); + UniValue witness(UniValue::VARR); + for (unsigned int i = 0; i < txin.scriptWitness.stack.size(); i++) { + witness.push_back(HexStr(txin.scriptWitness.stack[i].begin(), txin.scriptWitness.stack[i].end())); + } + entry.push_back(Pair("witness", witness)); entry.push_back(Pair("scriptSig", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); entry.push_back(Pair("sequence", (uint64_t)txin.nSequence)); entry.push_back(Pair("error", strMessage)); @@ -694,9 +638,7 @@ UniValue signrawtransaction(const JSONRPCRequest& request) view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view BOOST_FOREACH(const CTxIn& txin, mergedTx.vin) { - const uint256& prevHash = txin.prevout.hash; - CCoins coins; - view.AccessCoins(prevHash); // this is certainly allowed to fail + view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail. } view.SetBackend(viewDummy); // switch back to avoid locking mempool for too long @@ -748,24 +690,26 @@ UniValue signrawtransaction(const JSONRPCRequest& request) if (nOut < 0) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "vout must be positive"); + COutPoint out(txid, nOut); std::vector<unsigned char> pkData(ParseHexO(prevOut, "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); { - CCoinsModifier coins = view.ModifyCoins(txid); - if (coins->IsAvailable(nOut) && coins->vout[nOut].scriptPubKey != scriptPubKey) { + const Coin& coin = view.AccessCoin(out); + if (!coin.IsSpent() && coin.out.scriptPubKey != scriptPubKey) { std::string err("Previous output scriptPubKey mismatch:\n"); - err = err + ScriptToAsmStr(coins->vout[nOut].scriptPubKey) + "\nvs:\n"+ + err = err + ScriptToAsmStr(coin.out.scriptPubKey) + "\nvs:\n"+ ScriptToAsmStr(scriptPubKey); throw JSONRPCError(RPC_DESERIALIZATION_ERROR, err); } - if ((unsigned int)nOut >= coins->vout.size()) - coins->vout.resize(nOut+1); - coins->vout[nOut].scriptPubKey = scriptPubKey; - coins->vout[nOut].nValue = 0; + Coin newcoin; + newcoin.out.scriptPubKey = scriptPubKey; + newcoin.out.nValue = 0; if (prevOut.exists("amount")) { - coins->vout[nOut].nValue = AmountFromValue(find_value(prevOut, "amount")); + newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); } + newcoin.nHeight = 1; + view.AddCoin(out, std::move(newcoin), true); } // if redeemScript given and not using the local wallet (private keys @@ -823,13 +767,13 @@ UniValue signrawtransaction(const JSONRPCRequest& request) // Sign what we can: for (unsigned int i = 0; i < mergedTx.vin.size(); i++) { CTxIn& txin = mergedTx.vin[i]; - const CCoins* coins = view.AccessCoins(txin.prevout.hash); - if (coins == NULL || !coins->IsAvailable(txin.prevout.n)) { + const Coin& coin = view.AccessCoin(txin.prevout); + if (coin.IsSpent()) { TxInErrorToJSON(txin, vErrors, "Input not found or already spent"); continue; } - const CScript& prevPubKey = coins->vout[txin.prevout.n].scriptPubKey; - const CAmount& amount = coins->vout[txin.prevout.n].nValue; + const CScript& prevPubKey = coin.out.scriptPubKey; + const CAmount& amount = coin.out.nValue; SignatureData sigdata; // Only sign SIGHASH_SINGLE if there's a corresponding output: @@ -901,9 +845,12 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) nMaxRawTxFee = 0; CCoinsViewCache &view = *pcoinsTip; - const CCoins* existingCoins = view.AccessCoins(hashTx); + bool fHaveChain = false; + for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { + const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); + fHaveChain = !existingCoin.IsSpent(); + } bool fHaveMempool = mempool.exists(hashTx); - bool fHaveChain = existingCoins && existingCoins->nHeight < 1000000000; if (!fHaveMempool && !fHaveChain) { // push to local node and sync with wallets CValidationState state; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 5c493428d8..c5fbff0077 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -22,6 +22,8 @@ #include <boost/signals2/signal.hpp> #include <boost/thread.hpp> #include <boost/algorithm/string/case_conv.hpp> // for to_upper() +#include <boost/algorithm/string/classification.hpp> +#include <boost/algorithm/string/split.hpp> #include <memory> // for unique_ptr #include <unordered_map> @@ -42,17 +44,17 @@ static struct CRPCSignals boost::signals2::signal<void (const CRPCCommand&)> PreCommand; } g_rpcSignals; -void RPCServer::OnStarted(boost::function<void ()> slot) +void RPCServer::OnStarted(std::function<void ()> slot) { g_rpcSignals.Started.connect(slot); } -void RPCServer::OnStopped(boost::function<void ()> slot) +void RPCServer::OnStopped(std::function<void ()> slot) { g_rpcSignals.Stopped.connect(slot); } -void RPCServer::OnPreCommand(boost::function<void (const CRPCCommand&)> slot) +void RPCServer::OnPreCommand(std::function<void (const CRPCCommand&)> slot) { g_rpcSignals.PreCommand.connect(boost::bind(slot, _1)); } @@ -432,8 +434,16 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c } // Process expected parameters. int hole = 0; - for (const std::string &argName: argNames) { - auto fr = argsIn.find(argName); + for (const std::string &argNamePattern: argNames) { + std::vector<std::string> vargNames; + boost::algorithm::split(vargNames, argNamePattern, boost::algorithm::is_any_of("|")); + auto fr = argsIn.end(); + for (const std::string & argName : vargNames) { + fr = argsIn.find(argName); + if (fr != argsIn.end()) { + break; + } + } if (fr != argsIn.end()) { for (int i = 0; i < hole; ++i) { // Fill hole between specified parameters with JSON nulls, @@ -526,7 +536,7 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface) timerInterface = NULL; } -void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds) +void RPCRunLater(const std::string& name, std::function<void(void)> func, int64_t nSeconds) { if (!timerInterface) throw JSONRPCError(RPC_INTERNAL_ERROR, "No timer handler registered for RPC"); diff --git a/src/rpc/server.h b/src/rpc/server.h index 34a9d3c24c..1e984cbc0d 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -15,8 +15,6 @@ #include <stdint.h> #include <string> -#include <boost/function.hpp> - #include <univalue.h> static const unsigned int DEFAULT_RPC_SERIALIZE_VERSION = 1; @@ -25,9 +23,9 @@ class CRPCCommand; namespace RPCServer { - void OnStarted(boost::function<void ()> slot); - void OnStopped(boost::function<void ()> slot); - void OnPreCommand(boost::function<void (const CRPCCommand&)> slot); + void OnStarted(std::function<void ()> slot); + void OnStopped(std::function<void ()> slot); + void OnPreCommand(std::function<void (const CRPCCommand&)> slot); } class CBlockIndex; @@ -115,7 +113,7 @@ public: * This is needed to cope with the case in which there is no HTTP server, but * only GUI RPC console, and to break the dependency of pcserver on httprpc. */ - virtual RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) = 0; + virtual RPCTimerBase* NewTimer(std::function<void(void)>& func, int64_t millis) = 0; }; /** Set the factory function for timers */ @@ -129,7 +127,7 @@ void RPCUnsetTimerInterface(RPCTimerInterface *iface); * Run func nSeconds from now. * Overrides previous timer <name> (if any). */ -void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds); +void RPCRunLater(const std::string& name, std::function<void(void)> func, int64_t nSeconds); typedef UniValue(*rpcfn_type)(const JSONRPCRequest& jsonRequest); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 0c1cfa2718..923ba2c231 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -4,6 +4,7 @@ #include "scheduler.h" +#include "random.h" #include "reverselock.h" #include <assert.h> @@ -39,6 +40,11 @@ void CScheduler::serviceQueue() // is called. while (!shouldStop()) { try { + if (!shouldStop() && taskQueue.empty()) { + reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); + // Use this chance to get a tiny bit more entropy + RandAddSeedSleep(); + } while (!shouldStop() && taskQueue.empty()) { // Wait until there is something to do. newTaskScheduled.wait(lock); diff --git a/src/scheduler.h b/src/scheduler.h index 613fc1653f..27412a15b4 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -7,8 +7,8 @@ // // NOTE: -// boost::thread / boost::function / boost::chrono should be ported to -// std::thread / std::function / std::chrono when we support C++11. +// boost::thread / boost::chrono should be ported to std::thread / std::chrono +// when we support C++11. // #include <boost/chrono/chrono.hpp> #include <boost/thread.hpp> diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 8ecf0bbdac..222cff59ea 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -247,10 +247,10 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& { static const CScriptNum bnZero(0); static const CScriptNum bnOne(1); - static const CScriptNum bnFalse(0); - static const CScriptNum bnTrue(1); + // static const CScriptNum bnFalse(0); + // static const CScriptNum bnTrue(1); static const valtype vchFalse(0); - static const valtype vchZero(0); + // static const valtype vchZero(0); static const valtype vchTrue(1, 1); CScript::const_iterator pc = script.begin(); @@ -1028,7 +1028,7 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& } // Size limits - if (stack.size() + altstack.size() > 1000) + if (stack.size() + altstack.size() > MAX_STACK_SIZE) return set_error(serror, SCRIPT_ERR_STACK_SIZE); } } @@ -1142,24 +1142,24 @@ public: uint256 GetPrevoutHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vin.size(); n++) { - ss << txTo.vin[n].prevout; + for (const auto& txin : txTo.vin) { + ss << txin.prevout; } return ss.GetHash(); } uint256 GetSequenceHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vin.size(); n++) { - ss << txTo.vin[n].nSequence; + for (const auto& txin : txTo.vin) { + ss << txin.nSequence; } return ss.GetHash(); } uint256 GetOutputsHash(const CTransaction& txTo) { CHashWriter ss(SER_GETHASH, 0); - for (unsigned int n = 0; n < txTo.vout.size(); n++) { - ss << txTo.vout[n]; + for (const auto& txout : txTo.vout) { + ss << txout; } return ss.GetHash(); } diff --git a/src/script/script.h b/src/script/script.h index d7aaa04f80..95a5999a13 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -30,6 +30,9 @@ static const int MAX_PUBKEYS_PER_MULTISIG = 20; // Maximum script length in bytes static const int MAX_SCRIPT_SIZE = 10000; +// Maximum number of values on script interpreter stack +static const int MAX_STACK_SIZE = 1000; + // Threshold for nLockTime: below this value it is interpreted as block number, // otherwise as UNIX timestamp. static const unsigned int LOCKTIME_THRESHOLD = 500000000; // Tue Nov 5 00:53:20 1985 UTC diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 6f47b725fb..7bb8d9941b 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -15,28 +15,6 @@ #include <boost/thread.hpp> namespace { - -/** - * We're hashing a nonce into the entries themselves, so we don't need extra - * blinding in the set hash computation. - * - * This may exhibit platform endian dependent behavior but because these are - * nonced hashes (random) and this state is only ever used locally it is safe. - * All that matters is local consistency. - */ -class SignatureCacheHasher -{ -public: - template <uint8_t hash_select> - uint32_t operator()(const uint256& key) const - { - static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); - uint32_t u; - std::memcpy(&u, key.begin()+4*hash_select, 4); - return u; - } -}; - /** * Valid signature cache, to avoid doing expensive ECDSA signature checking * twice for every transaction (once when accepted into memory pool, and diff --git a/src/script/sigcache.h b/src/script/sigcache.h index 60690583de..55cec4cc8d 100644 --- a/src/script/sigcache.h +++ b/src/script/sigcache.h @@ -19,6 +19,27 @@ static const int64_t MAX_MAX_SIG_CACHE_SIZE = 16384; class CPubKey; +/** + * We're hashing a nonce into the entries themselves, so we don't need extra + * blinding in the set hash computation. + * + * This may exhibit platform endian dependent behavior but because these are + * nonced hashes (random) and this state is only ever used locally it is safe. + * All that matters is local consistency. + */ +class SignatureCacheHasher +{ +public: + template <uint8_t hash_select> + uint32_t operator()(const uint256& key) const + { + static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); + uint32_t u; + std::memcpy(&u, key.begin()+4*hash_select, 4); + return u; + } +}; + class CachingTransactionSignatureChecker : public TransactionSignatureChecker { private: diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index e5657f7f31..c071fbe275 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -93,7 +93,10 @@ TESTS = if USE_TESTS noinst_PROGRAMS += tests tests_SOURCES = src/tests.c -tests_CPPFLAGS = -DSECP256K1_BUILD -DVERIFY -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) +tests_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) +if !ENABLE_COVERAGE +tests_CPPFLAGS += -DVERIFY +endif tests_LDADD = $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) tests_LDFLAGS = -static TESTS += tests @@ -102,7 +105,10 @@ endif if USE_EXHAUSTIVE_TESTS noinst_PROGRAMS += exhaustive_tests exhaustive_tests_SOURCES = src/tests_exhaustive.c -exhaustive_tests_CPPFLAGS = -DSECP256K1_BUILD -DVERIFY -I$(top_srcdir)/src $(SECP_INCLUDES) +exhaustive_tests_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/src $(SECP_INCLUDES) +if !ENABLE_COVERAGE +exhaustive_tests_CPPFLAGS += -DVERIFY +endif exhaustive_tests_LDADD = $(SECP_LIBS) exhaustive_tests_LDFLAGS = -static TESTS += exhaustive_tests diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index ec50ffe3a2..e5fcbcb4ed 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -20,7 +20,7 @@ AC_PATH_TOOL(STRIP, strip) AX_PROG_CC_FOR_BUILD if test "x$CFLAGS" = "x"; then - CFLAGS="-O3 -g" + CFLAGS="-g" fi AM_PROG_CC_C_O @@ -89,6 +89,11 @@ AC_ARG_ENABLE(benchmark, [use_benchmark=$enableval], [use_benchmark=no]) +AC_ARG_ENABLE(coverage, + AS_HELP_STRING([--enable-coverage],[enable compiler flags to support kcov coverage analysis]), + [enable_coverage=$enableval], + [enable_coverage=no]) + AC_ARG_ENABLE(tests, AS_HELP_STRING([--enable-tests],[compile tests (default is yes)]), [use_tests=$enableval], @@ -154,6 +159,14 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[void myfunc() {__builtin_expect(0,0);}]])], [ AC_MSG_RESULT([no]) ]) +if test x"$enable_coverage" = x"yes"; then + AC_DEFINE(COVERAGE, 1, [Define this symbol to compile out all VERIFY code]) + CFLAGS="$CFLAGS -O0 --coverage" + LDFLAGS="--coverage" +else + CFLAGS="$CFLAGS -O3" +fi + if test x"$use_ecmult_static_precomputation" != x"no"; then save_cross_compiling=$cross_compiling cross_compiling=no @@ -434,6 +447,7 @@ AC_MSG_NOTICE([Using field implementation: $set_field]) AC_MSG_NOTICE([Using bignum implementation: $set_bignum]) AC_MSG_NOTICE([Using scalar implementation: $set_scalar]) AC_MSG_NOTICE([Using endomorphism optimizations: $use_endomorphism]) +AC_MSG_NOTICE([Building for coverage analysis: $enable_coverage]) AC_MSG_NOTICE([Building ECDH module: $enable_module_ecdh]) AC_MSG_NOTICE([Building ECDSA pubkey recovery module: $enable_module_recovery]) AC_MSG_NOTICE([Using jni: $use_jni]) @@ -460,6 +474,7 @@ AC_SUBST(SECP_INCLUDES) AC_SUBST(SECP_LIBS) AC_SUBST(SECP_TEST_LIBS) AC_SUBST(SECP_TEST_INCLUDES) +AM_CONDITIONAL([ENABLE_COVERAGE], [test x"$enable_coverage" = x"yes"]) AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" != x"no"]) AM_CONDITIONAL([USE_EXHAUSTIVE_TESTS], [test x"$use_exhaustive_tests" != x"no"]) AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"]) diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index f268e309d0..fc4c5cefbb 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -163,6 +163,8 @@ typedef int (*secp256k1_nonce_function)( * * Returns: a newly created context object. * In: flags: which parts of the context to initialize. + * + * See also secp256k1_context_randomize. */ SECP256K1_API secp256k1_context* secp256k1_context_create( unsigned int flags @@ -485,6 +487,28 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_create( const unsigned char *seckey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +/** Negates a private key in place. + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * In/Out: pubkey: pointer to the public key to be negated (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_privkey_negate( + const secp256k1_context* ctx, + unsigned char *seckey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + +/** Negates a public key in place. + * + * Returns: 1 always + * Args: ctx: pointer to a context object + * In/Out: pubkey: pointer to the public key to be negated (cannot be NULL) + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_negate( + const secp256k1_context* ctx, + secp256k1_pubkey *pubkey +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2); + /** Tweak a private key by adding tweak to it. * Returns: 0 if the tweak was out of range (chance of around 1 in 2^128 for * uniformly random 32-byte arrays, or if the resulting private key @@ -543,11 +567,24 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_tweak_mul( const unsigned char *tweak ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); -/** Updates the context randomization. +/** Updates the context randomization to protect against side-channel leakage. * Returns: 1: randomization successfully updated * 0: error * Args: ctx: pointer to a context object (cannot be NULL) * In: seed32: pointer to a 32-byte random seed (NULL resets to initial state) + * + * While secp256k1 code is written to be constant-time no matter what secret + * values are, it's possible that a future compiler may output code which isn't, + * and also that the CPU may not emit the same radio frequencies or draw the same + * amount power for all values. + * + * This function provides a seed which is combined into the blinding value: that + * blinding value is added before each multiplication (and removed afterwards) so + * that it does not affect function results, but shields against attacks which + * rely on any input-dependent behaviour. + * + * You should call this after secp256k1_context_create or + * secp256k1_context_clone, and may call this repeatedly afterwards. */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_context_randomize( secp256k1_context* ctx, diff --git a/src/secp256k1/src/bench.h b/src/secp256k1/src/bench.h index 3a71b4aafa..d67f08a426 100644 --- a/src/secp256k1/src/bench.h +++ b/src/secp256k1/src/bench.h @@ -23,7 +23,7 @@ void print_number(double x) { if (y < 0.0) { y = -y; } - while (y < 100.0) { + while (y > 0 && y < 100.0) { y *= 10.0; c++; } diff --git a/src/secp256k1/src/bench_schnorr_verify.c b/src/secp256k1/src/bench_schnorr_verify.c deleted file mode 100644 index 5f137dda23..0000000000 --- a/src/secp256k1/src/bench_schnorr_verify.c +++ /dev/null @@ -1,73 +0,0 @@ -/********************************************************************** - * Copyright (c) 2014 Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or http://www.opensource.org/licenses/mit-license.php.* - **********************************************************************/ - -#include <stdio.h> -#include <string.h> - -#include "include/secp256k1.h" -#include "include/secp256k1_schnorr.h" -#include "util.h" -#include "bench.h" - -typedef struct { - unsigned char key[32]; - unsigned char sig[64]; - unsigned char pubkey[33]; - size_t pubkeylen; -} benchmark_schnorr_sig_t; - -typedef struct { - secp256k1_context *ctx; - unsigned char msg[32]; - benchmark_schnorr_sig_t sigs[64]; - int numsigs; -} benchmark_schnorr_verify_t; - -static void benchmark_schnorr_init(void* arg) { - int i, k; - benchmark_schnorr_verify_t* data = (benchmark_schnorr_verify_t*)arg; - - for (i = 0; i < 32; i++) { - data->msg[i] = 1 + i; - } - for (k = 0; k < data->numsigs; k++) { - secp256k1_pubkey pubkey; - for (i = 0; i < 32; i++) { - data->sigs[k].key[i] = 33 + i + k; - } - secp256k1_schnorr_sign(data->ctx, data->sigs[k].sig, data->msg, data->sigs[k].key, NULL, NULL); - data->sigs[k].pubkeylen = 33; - CHECK(secp256k1_ec_pubkey_create(data->ctx, &pubkey, data->sigs[k].key)); - CHECK(secp256k1_ec_pubkey_serialize(data->ctx, data->sigs[k].pubkey, &data->sigs[k].pubkeylen, &pubkey, SECP256K1_EC_COMPRESSED)); - } -} - -static void benchmark_schnorr_verify(void* arg) { - int i; - benchmark_schnorr_verify_t* data = (benchmark_schnorr_verify_t*)arg; - - for (i = 0; i < 20000 / data->numsigs; i++) { - secp256k1_pubkey pubkey; - data->sigs[0].sig[(i >> 8) % 64] ^= (i & 0xFF); - CHECK(secp256k1_ec_pubkey_parse(data->ctx, &pubkey, data->sigs[0].pubkey, data->sigs[0].pubkeylen)); - CHECK(secp256k1_schnorr_verify(data->ctx, data->sigs[0].sig, data->msg, &pubkey) == ((i & 0xFF) == 0)); - data->sigs[0].sig[(i >> 8) % 64] ^= (i & 0xFF); - } -} - - - -int main(void) { - benchmark_schnorr_verify_t data; - - data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - - data.numsigs = 1; - run_benchmark("schnorr_verify", benchmark_schnorr_verify, benchmark_schnorr_init, NULL, &data, 10, 20000); - - secp256k1_context_destroy(data.ctx); - return 0; -} diff --git a/src/secp256k1/src/ecdsa_impl.h b/src/secp256k1/src/ecdsa_impl.h index 9a42e519bd..453bb11880 100644 --- a/src/secp256k1/src/ecdsa_impl.h +++ b/src/secp256k1/src/ecdsa_impl.h @@ -225,14 +225,12 @@ static int secp256k1_ecdsa_sig_verify(const secp256k1_ecmult_context *ctx, const #if defined(EXHAUSTIVE_TEST_ORDER) { secp256k1_scalar computed_r; - int overflow = 0; secp256k1_ge pr_ge; secp256k1_ge_set_gej(&pr_ge, &pr); secp256k1_fe_normalize(&pr_ge.x); secp256k1_fe_get_b32(c, &pr_ge.x); - secp256k1_scalar_set_b32(&computed_r, c, &overflow); - /* we fully expect overflow */ + secp256k1_scalar_set_b32(&computed_r, c, NULL); return secp256k1_scalar_eq(sigr, &computed_r); } #else @@ -285,14 +283,10 @@ static int secp256k1_ecdsa_sig_sign(const secp256k1_ecmult_gen_context *ctx, sec secp256k1_fe_normalize(&r.y); secp256k1_fe_get_b32(b, &r.x); secp256k1_scalar_set_b32(sigr, b, &overflow); - if (secp256k1_scalar_is_zero(sigr)) { - /* P.x = order is on the curve, so technically sig->r could end up zero, which would be an invalid signature. - * This branch is cryptographically unreachable as hitting it requires finding the discrete log of P.x = N. - */ - secp256k1_gej_clear(&rp); - secp256k1_ge_clear(&r); - return 0; - } + /* These two conditions should be checked before calling */ + VERIFY_CHECK(!secp256k1_scalar_is_zero(sigr)); + VERIFY_CHECK(overflow == 0); + if (recid) { /* The overflow condition is cryptographically unreachable as hitting it requires finding the discrete log * of some P where P.x >= order, and only 1 in about 2^127 points meet this criteria. diff --git a/src/secp256k1/src/field_10x26_impl.h b/src/secp256k1/src/field_10x26_impl.h index 7b8c079608..234c13a644 100644 --- a/src/secp256k1/src/field_10x26_impl.h +++ b/src/secp256k1/src/field_10x26_impl.h @@ -38,10 +38,6 @@ static void secp256k1_fe_verify(const secp256k1_fe *a) { } VERIFY_CHECK(r == 1); } -#else -static void secp256k1_fe_verify(const secp256k1_fe *a) { - (void)a; -} #endif static void secp256k1_fe_normalize(secp256k1_fe *r) { @@ -325,17 +321,17 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { } static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { - int i; - r->n[0] = r->n[1] = r->n[2] = r->n[3] = r->n[4] = 0; - r->n[5] = r->n[6] = r->n[7] = r->n[8] = r->n[9] = 0; - for (i=0; i<32; i++) { - int j; - for (j=0; j<4; j++) { - int limb = (8*i+2*j)/26; - int shift = (8*i+2*j)%26; - r->n[limb] |= (uint32_t)((a[31-i] >> (2*j)) & 0x3) << shift; - } - } + r->n[0] = (uint32_t)a[31] | ((uint32_t)a[30] << 8) | ((uint32_t)a[29] << 16) | ((uint32_t)(a[28] & 0x3) << 24); + r->n[1] = (uint32_t)((a[28] >> 2) & 0x3f) | ((uint32_t)a[27] << 6) | ((uint32_t)a[26] << 14) | ((uint32_t)(a[25] & 0xf) << 22); + r->n[2] = (uint32_t)((a[25] >> 4) & 0xf) | ((uint32_t)a[24] << 4) | ((uint32_t)a[23] << 12) | ((uint32_t)(a[22] & 0x3f) << 20); + r->n[3] = (uint32_t)((a[22] >> 6) & 0x3) | ((uint32_t)a[21] << 2) | ((uint32_t)a[20] << 10) | ((uint32_t)a[19] << 18); + r->n[4] = (uint32_t)a[18] | ((uint32_t)a[17] << 8) | ((uint32_t)a[16] << 16) | ((uint32_t)(a[15] & 0x3) << 24); + r->n[5] = (uint32_t)((a[15] >> 2) & 0x3f) | ((uint32_t)a[14] << 6) | ((uint32_t)a[13] << 14) | ((uint32_t)(a[12] & 0xf) << 22); + r->n[6] = (uint32_t)((a[12] >> 4) & 0xf) | ((uint32_t)a[11] << 4) | ((uint32_t)a[10] << 12) | ((uint32_t)(a[9] & 0x3f) << 20); + r->n[7] = (uint32_t)((a[9] >> 6) & 0x3) | ((uint32_t)a[8] << 2) | ((uint32_t)a[7] << 10) | ((uint32_t)a[6] << 18); + r->n[8] = (uint32_t)a[5] | ((uint32_t)a[4] << 8) | ((uint32_t)a[3] << 16) | ((uint32_t)(a[2] & 0x3) << 24); + r->n[9] = (uint32_t)((a[2] >> 2) & 0x3f) | ((uint32_t)a[1] << 6) | ((uint32_t)a[0] << 14); + if (r->n[9] == 0x3FFFFFUL && (r->n[8] & r->n[7] & r->n[6] & r->n[5] & r->n[4] & r->n[3] & r->n[2]) == 0x3FFFFFFUL && (r->n[1] + 0x40UL + ((r->n[0] + 0x3D1UL) >> 26)) > 0x3FFFFFFUL) { return 0; } @@ -349,21 +345,42 @@ static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { - int i; #ifdef VERIFY VERIFY_CHECK(a->normalized); secp256k1_fe_verify(a); #endif - for (i=0; i<32; i++) { - int j; - int c = 0; - for (j=0; j<4; j++) { - int limb = (8*i+2*j)/26; - int shift = (8*i+2*j)%26; - c |= ((a->n[limb] >> shift) & 0x3) << (2 * j); - } - r[31-i] = c; - } + r[0] = (a->n[9] >> 14) & 0xff; + r[1] = (a->n[9] >> 6) & 0xff; + r[2] = ((a->n[9] & 0x3F) << 2) | ((a->n[8] >> 24) & 0x3); + r[3] = (a->n[8] >> 16) & 0xff; + r[4] = (a->n[8] >> 8) & 0xff; + r[5] = a->n[8] & 0xff; + r[6] = (a->n[7] >> 18) & 0xff; + r[7] = (a->n[7] >> 10) & 0xff; + r[8] = (a->n[7] >> 2) & 0xff; + r[9] = ((a->n[7] & 0x3) << 6) | ((a->n[6] >> 20) & 0x3f); + r[10] = (a->n[6] >> 12) & 0xff; + r[11] = (a->n[6] >> 4) & 0xff; + r[12] = ((a->n[6] & 0xf) << 4) | ((a->n[5] >> 22) & 0xf); + r[13] = (a->n[5] >> 14) & 0xff; + r[14] = (a->n[5] >> 6) & 0xff; + r[15] = ((a->n[5] & 0x3f) << 2) | ((a->n[4] >> 24) & 0x3); + r[16] = (a->n[4] >> 16) & 0xff; + r[17] = (a->n[4] >> 8) & 0xff; + r[18] = a->n[4] & 0xff; + r[19] = (a->n[3] >> 18) & 0xff; + r[20] = (a->n[3] >> 10) & 0xff; + r[21] = (a->n[3] >> 2) & 0xff; + r[22] = ((a->n[3] & 0x3) << 6) | ((a->n[2] >> 20) & 0x3f); + r[23] = (a->n[2] >> 12) & 0xff; + r[24] = (a->n[2] >> 4) & 0xff; + r[25] = ((a->n[2] & 0xf) << 4) | ((a->n[1] >> 22) & 0xf); + r[26] = (a->n[1] >> 14) & 0xff; + r[27] = (a->n[1] >> 6) & 0xff; + r[28] = ((a->n[1] & 0x3f) << 2) | ((a->n[0] >> 24) & 0x3); + r[29] = (a->n[0] >> 16) & 0xff; + r[30] = (a->n[0] >> 8) & 0xff; + r[31] = a->n[0] & 0xff; } SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { diff --git a/src/secp256k1/src/field_5x52_impl.h b/src/secp256k1/src/field_5x52_impl.h index 7a99eb21ec..8e8b286baf 100644 --- a/src/secp256k1/src/field_5x52_impl.h +++ b/src/secp256k1/src/field_5x52_impl.h @@ -49,10 +49,6 @@ static void secp256k1_fe_verify(const secp256k1_fe *a) { } VERIFY_CHECK(r == 1); } -#else -static void secp256k1_fe_verify(const secp256k1_fe *a) { - (void)a; -} #endif static void secp256k1_fe_normalize(secp256k1_fe *r) { @@ -288,16 +284,40 @@ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { } static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { - int i; - r->n[0] = r->n[1] = r->n[2] = r->n[3] = r->n[4] = 0; - for (i=0; i<32; i++) { - int j; - for (j=0; j<2; j++) { - int limb = (8*i+4*j)/52; - int shift = (8*i+4*j)%52; - r->n[limb] |= (uint64_t)((a[31-i] >> (4*j)) & 0xF) << shift; - } - } + r->n[0] = (uint64_t)a[31] + | ((uint64_t)a[30] << 8) + | ((uint64_t)a[29] << 16) + | ((uint64_t)a[28] << 24) + | ((uint64_t)a[27] << 32) + | ((uint64_t)a[26] << 40) + | ((uint64_t)(a[25] & 0xF) << 48); + r->n[1] = (uint64_t)((a[25] >> 4) & 0xF) + | ((uint64_t)a[24] << 4) + | ((uint64_t)a[23] << 12) + | ((uint64_t)a[22] << 20) + | ((uint64_t)a[21] << 28) + | ((uint64_t)a[20] << 36) + | ((uint64_t)a[19] << 44); + r->n[2] = (uint64_t)a[18] + | ((uint64_t)a[17] << 8) + | ((uint64_t)a[16] << 16) + | ((uint64_t)a[15] << 24) + | ((uint64_t)a[14] << 32) + | ((uint64_t)a[13] << 40) + | ((uint64_t)(a[12] & 0xF) << 48); + r->n[3] = (uint64_t)((a[12] >> 4) & 0xF) + | ((uint64_t)a[11] << 4) + | ((uint64_t)a[10] << 12) + | ((uint64_t)a[9] << 20) + | ((uint64_t)a[8] << 28) + | ((uint64_t)a[7] << 36) + | ((uint64_t)a[6] << 44); + r->n[4] = (uint64_t)a[5] + | ((uint64_t)a[4] << 8) + | ((uint64_t)a[3] << 16) + | ((uint64_t)a[2] << 24) + | ((uint64_t)a[1] << 32) + | ((uint64_t)a[0] << 40); if (r->n[4] == 0x0FFFFFFFFFFFFULL && (r->n[3] & r->n[2] & r->n[1]) == 0xFFFFFFFFFFFFFULL && r->n[0] >= 0xFFFFEFFFFFC2FULL) { return 0; } @@ -311,21 +331,42 @@ static int secp256k1_fe_set_b32(secp256k1_fe *r, const unsigned char *a) { /** Convert a field element to a 32-byte big endian value. Requires the input to be normalized */ static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { - int i; #ifdef VERIFY VERIFY_CHECK(a->normalized); secp256k1_fe_verify(a); #endif - for (i=0; i<32; i++) { - int j; - int c = 0; - for (j=0; j<2; j++) { - int limb = (8*i+4*j)/52; - int shift = (8*i+4*j)%52; - c |= ((a->n[limb] >> shift) & 0xF) << (4 * j); - } - r[31-i] = c; - } + r[0] = (a->n[4] >> 40) & 0xFF; + r[1] = (a->n[4] >> 32) & 0xFF; + r[2] = (a->n[4] >> 24) & 0xFF; + r[3] = (a->n[4] >> 16) & 0xFF; + r[4] = (a->n[4] >> 8) & 0xFF; + r[5] = a->n[4] & 0xFF; + r[6] = (a->n[3] >> 44) & 0xFF; + r[7] = (a->n[3] >> 36) & 0xFF; + r[8] = (a->n[3] >> 28) & 0xFF; + r[9] = (a->n[3] >> 20) & 0xFF; + r[10] = (a->n[3] >> 12) & 0xFF; + r[11] = (a->n[3] >> 4) & 0xFF; + r[12] = ((a->n[2] >> 48) & 0xF) | ((a->n[3] & 0xF) << 4); + r[13] = (a->n[2] >> 40) & 0xFF; + r[14] = (a->n[2] >> 32) & 0xFF; + r[15] = (a->n[2] >> 24) & 0xFF; + r[16] = (a->n[2] >> 16) & 0xFF; + r[17] = (a->n[2] >> 8) & 0xFF; + r[18] = a->n[2] & 0xFF; + r[19] = (a->n[1] >> 44) & 0xFF; + r[20] = (a->n[1] >> 36) & 0xFF; + r[21] = (a->n[1] >> 28) & 0xFF; + r[22] = (a->n[1] >> 20) & 0xFF; + r[23] = (a->n[1] >> 12) & 0xFF; + r[24] = (a->n[1] >> 4) & 0xFF; + r[25] = ((a->n[0] >> 48) & 0xF) | ((a->n[1] & 0xF) << 4); + r[26] = (a->n[0] >> 40) & 0xFF; + r[27] = (a->n[0] >> 32) & 0xFF; + r[28] = (a->n[0] >> 24) & 0xFF; + r[29] = (a->n[0] >> 16) & 0xFF; + r[30] = (a->n[0] >> 8) & 0xFF; + r[31] = a->n[0] & 0xFF; } SECP256K1_INLINE static void secp256k1_fe_negate(secp256k1_fe *r, const secp256k1_fe *a, int m) { diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 2e192b62fd..7d723532ff 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -200,12 +200,6 @@ static void secp256k1_gej_set_infinity(secp256k1_gej *r) { secp256k1_fe_clear(&r->z); } -static void secp256k1_ge_set_infinity(secp256k1_ge *r) { - r->infinity = 1; - secp256k1_fe_clear(&r->x); - secp256k1_fe_clear(&r->y); -} - static void secp256k1_gej_clear(secp256k1_gej *r) { r->infinity = 0; secp256k1_fe_clear(&r->x); diff --git a/src/secp256k1/src/modules/ecdh/main_impl.h b/src/secp256k1/src/modules/ecdh/main_impl.h index c23e4f82f7..9e30fb73dd 100644 --- a/src/secp256k1/src/modules/ecdh/main_impl.h +++ b/src/secp256k1/src/modules/ecdh/main_impl.h @@ -16,10 +16,10 @@ int secp256k1_ecdh(const secp256k1_context* ctx, unsigned char *result, const se secp256k1_gej res; secp256k1_ge pt; secp256k1_scalar s; + VERIFY_CHECK(ctx != NULL); ARG_CHECK(result != NULL); ARG_CHECK(point != NULL); ARG_CHECK(scalar != NULL); - (void)ctx; secp256k1_pubkey_load(ctx, &pt, point); secp256k1_scalar_set_b32(&s, scalar, &overflow); diff --git a/src/secp256k1/src/modules/ecdh/tests_impl.h b/src/secp256k1/src/modules/ecdh/tests_impl.h index 7badc9033f..85a5d0a9a6 100644 --- a/src/secp256k1/src/modules/ecdh/tests_impl.h +++ b/src/secp256k1/src/modules/ecdh/tests_impl.h @@ -7,6 +7,35 @@ #ifndef _SECP256K1_MODULE_ECDH_TESTS_ #define _SECP256K1_MODULE_ECDH_TESTS_ +void test_ecdh_api(void) { + /* Setup context that just counts errors */ + secp256k1_context *tctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + secp256k1_pubkey point; + unsigned char res[32]; + unsigned char s_one[32] = { 0 }; + int32_t ecount = 0; + s_one[31] = 1; + + secp256k1_context_set_error_callback(tctx, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(tctx, counting_illegal_callback_fn, &ecount); + CHECK(secp256k1_ec_pubkey_create(tctx, &point, s_one) == 1); + + /* Check all NULLs are detected */ + CHECK(secp256k1_ecdh(tctx, res, &point, s_one) == 1); + CHECK(ecount == 0); + CHECK(secp256k1_ecdh(tctx, NULL, &point, s_one) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdh(tctx, res, NULL, s_one) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdh(tctx, res, &point, NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdh(tctx, res, &point, s_one) == 1); + CHECK(ecount == 3); + + /* Cleanup */ + secp256k1_context_destroy(tctx); +} + void test_ecdh_generator_basepoint(void) { unsigned char s_one[32] = { 0 }; secp256k1_pubkey point[2]; @@ -68,6 +97,7 @@ void test_bad_scalar(void) { } void run_ecdh_tests(void) { + test_ecdh_api(); test_ecdh_generator_basepoint(); test_bad_scalar(); } diff --git a/src/secp256k1/src/modules/recovery/main_impl.h b/src/secp256k1/src/modules/recovery/main_impl.h index 86f2f0cb2b..c6fbe23981 100755 --- a/src/secp256k1/src/modules/recovery/main_impl.h +++ b/src/secp256k1/src/modules/recovery/main_impl.h @@ -179,7 +179,7 @@ int secp256k1_ecdsa_recover(const secp256k1_context* ctx, secp256k1_pubkey *pubk ARG_CHECK(pubkey != NULL); secp256k1_ecdsa_recoverable_signature_load(ctx, &r, &s, &recid, signature); - ARG_CHECK(recid >= 0 && recid < 4); + VERIFY_CHECK(recid >= 0 && recid < 4); /* should have been caught in parse_compact */ secp256k1_scalar_set_b32(&m, msg32, NULL); if (secp256k1_ecdsa_sig_recover(&ctx->ecmult_ctx, &r, &s, &q, &m, recid)) { secp256k1_pubkey_save(pubkey, &q); diff --git a/src/secp256k1/src/modules/recovery/tests_impl.h b/src/secp256k1/src/modules/recovery/tests_impl.h index 8932d5f0af..765c7dd81e 100644 --- a/src/secp256k1/src/modules/recovery/tests_impl.h +++ b/src/secp256k1/src/modules/recovery/tests_impl.h @@ -7,6 +7,146 @@ #ifndef _SECP256K1_MODULE_RECOVERY_TESTS_ #define _SECP256K1_MODULE_RECOVERY_TESTS_ +static int recovery_test_nonce_function(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *algo16, void *data, unsigned int counter) { + (void) msg32; + (void) key32; + (void) algo16; + (void) data; + + /* On the first run, return 0 to force a second run */ + if (counter == 0) { + memset(nonce32, 0, 32); + return 1; + } + /* On the second run, return an overflow to force a third run */ + if (counter == 1) { + memset(nonce32, 0xff, 32); + return 1; + } + /* On the next run, return a valid nonce, but flip a coin as to whether or not to fail signing. */ + memset(nonce32, 1, 32); + return secp256k1_rand_bits(1); +} + +void test_ecdsa_recovery_api(void) { + /* Setup contexts that just count errors */ + secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); + secp256k1_context *sign = secp256k1_context_create(SECP256K1_CONTEXT_SIGN); + secp256k1_context *vrfy = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); + secp256k1_context *both = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + secp256k1_pubkey pubkey; + secp256k1_pubkey recpubkey; + secp256k1_ecdsa_signature normal_sig; + secp256k1_ecdsa_recoverable_signature recsig; + unsigned char privkey[32] = { 1 }; + unsigned char message[32] = { 2 }; + int32_t ecount = 0; + int recid = 0; + unsigned char sig[74]; + unsigned char zero_privkey[32] = { 0 }; + unsigned char over_privkey[32] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + + secp256k1_context_set_error_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(sign, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(vrfy, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_error_callback(both, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(none, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(sign, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(vrfy, counting_illegal_callback_fn, &ecount); + secp256k1_context_set_illegal_callback(both, counting_illegal_callback_fn, &ecount); + + /* Construct and verify corresponding public key. */ + CHECK(secp256k1_ec_seckey_verify(ctx, privkey) == 1); + CHECK(secp256k1_ec_pubkey_create(ctx, &pubkey, privkey) == 1); + + /* Check bad contexts and NULLs for signing */ + ecount = 0; + CHECK(secp256k1_ecdsa_sign_recoverable(none, &recsig, message, privkey, NULL, NULL) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_sign_recoverable(sign, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_sign_recoverable(vrfy, &recsig, message, privkey, NULL, NULL) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_sign_recoverable(both, NULL, message, privkey, NULL, NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, NULL, privkey, NULL, NULL) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, NULL, NULL, NULL) == 0); + CHECK(ecount == 5); + /* This will fail or succeed randomly, and in either case will not ARG_CHECK failure */ + secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, recovery_test_nonce_function, NULL); + CHECK(ecount == 5); + /* These will all fail, but not in ARG_CHECK way */ + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, zero_privkey, NULL, NULL) == 0); + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, over_privkey, NULL, NULL) == 0); + /* This one will succeed. */ + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + CHECK(ecount == 5); + + /* Check signing with a goofy nonce function */ + + /* Check bad contexts and NULLs for recovery */ + ecount = 0; + CHECK(secp256k1_ecdsa_recover(none, &recpubkey, &recsig, message) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_recover(sign, &recpubkey, &recsig, message) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recover(vrfy, &recpubkey, &recsig, message) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recover(both, &recpubkey, &recsig, message) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recover(both, NULL, &recsig, message) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdsa_recover(both, &recpubkey, NULL, message) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_ecdsa_recover(both, &recpubkey, &recsig, NULL) == 0); + CHECK(ecount == 5); + + /* Check NULLs for conversion */ + CHECK(secp256k1_ecdsa_sign(both, &normal_sig, message, privkey, NULL, NULL) == 1); + ecount = 0; + CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, NULL, &recsig) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, &normal_sig, NULL) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recoverable_signature_convert(both, &normal_sig, &recsig) == 1); + + /* Check NULLs for de/serialization */ + CHECK(secp256k1_ecdsa_sign_recoverable(both, &recsig, message, privkey, NULL, NULL) == 1); + ecount = 0; + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, NULL, &recid, &recsig) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, NULL, &recsig) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, &recid, NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(both, sig, &recid, &recsig) == 1); + + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, NULL, sig, recid) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, NULL, recid) == 0); + CHECK(ecount == 5); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, -1) == 0); + CHECK(ecount == 6); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, 5) == 0); + CHECK(ecount == 7); + /* overflow in signature will fail but not affect ecount */ + memcpy(sig, over_privkey, 32); + CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(both, &recsig, sig, recid) == 0); + CHECK(ecount == 7); + + /* cleanup */ + secp256k1_context_destroy(none); + secp256k1_context_destroy(sign); + secp256k1_context_destroy(vrfy); + secp256k1_context_destroy(both); +} + void test_ecdsa_recovery_end_to_end(void) { unsigned char extra[32] = {0x00}; unsigned char privkey[32]; @@ -241,6 +381,9 @@ void test_ecdsa_recovery_edge_cases(void) { void run_recovery_tests(void) { int i; + for (i = 0; i < count; i++) { + test_ecdsa_recovery_api(); + } for (i = 0; i < 64*count; i++) { test_ecdsa_recovery_end_to_end(); } diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index f5b2376407..2690d86558 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -66,88 +66,79 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar #else secp256k1_scalar *t; int i; - /* First compute x ^ (2^N - 1) for some values of N. */ - secp256k1_scalar x2, x3, x4, x6, x7, x8, x15, x30, x60, x120, x127; + /* First compute xN as x ^ (2^N - 1) for some values of N, + * and uM as x ^ M for some values of M. */ + secp256k1_scalar x2, x3, x6, x8, x14, x28, x56, x112, x126; + secp256k1_scalar u2, u5, u9, u11, u13; - secp256k1_scalar_sqr(&x2, x); - secp256k1_scalar_mul(&x2, &x2, x); + secp256k1_scalar_sqr(&u2, x); + secp256k1_scalar_mul(&x2, &u2, x); + secp256k1_scalar_mul(&u5, &u2, &x2); + secp256k1_scalar_mul(&x3, &u5, &u2); + secp256k1_scalar_mul(&u9, &x3, &u2); + secp256k1_scalar_mul(&u11, &u9, &u2); + secp256k1_scalar_mul(&u13, &u11, &u2); - secp256k1_scalar_sqr(&x3, &x2); - secp256k1_scalar_mul(&x3, &x3, x); - - secp256k1_scalar_sqr(&x4, &x3); - secp256k1_scalar_mul(&x4, &x4, x); - - secp256k1_scalar_sqr(&x6, &x4); + secp256k1_scalar_sqr(&x6, &u13); secp256k1_scalar_sqr(&x6, &x6); - secp256k1_scalar_mul(&x6, &x6, &x2); - - secp256k1_scalar_sqr(&x7, &x6); - secp256k1_scalar_mul(&x7, &x7, x); + secp256k1_scalar_mul(&x6, &x6, &u11); - secp256k1_scalar_sqr(&x8, &x7); - secp256k1_scalar_mul(&x8, &x8, x); + secp256k1_scalar_sqr(&x8, &x6); + secp256k1_scalar_sqr(&x8, &x8); + secp256k1_scalar_mul(&x8, &x8, &x2); - secp256k1_scalar_sqr(&x15, &x8); - for (i = 0; i < 6; i++) { - secp256k1_scalar_sqr(&x15, &x15); + secp256k1_scalar_sqr(&x14, &x8); + for (i = 0; i < 5; i++) { + secp256k1_scalar_sqr(&x14, &x14); } - secp256k1_scalar_mul(&x15, &x15, &x7); + secp256k1_scalar_mul(&x14, &x14, &x6); - secp256k1_scalar_sqr(&x30, &x15); - for (i = 0; i < 14; i++) { - secp256k1_scalar_sqr(&x30, &x30); + secp256k1_scalar_sqr(&x28, &x14); + for (i = 0; i < 13; i++) { + secp256k1_scalar_sqr(&x28, &x28); } - secp256k1_scalar_mul(&x30, &x30, &x15); + secp256k1_scalar_mul(&x28, &x28, &x14); - secp256k1_scalar_sqr(&x60, &x30); - for (i = 0; i < 29; i++) { - secp256k1_scalar_sqr(&x60, &x60); + secp256k1_scalar_sqr(&x56, &x28); + for (i = 0; i < 27; i++) { + secp256k1_scalar_sqr(&x56, &x56); } - secp256k1_scalar_mul(&x60, &x60, &x30); + secp256k1_scalar_mul(&x56, &x56, &x28); - secp256k1_scalar_sqr(&x120, &x60); - for (i = 0; i < 59; i++) { - secp256k1_scalar_sqr(&x120, &x120); + secp256k1_scalar_sqr(&x112, &x56); + for (i = 0; i < 55; i++) { + secp256k1_scalar_sqr(&x112, &x112); } - secp256k1_scalar_mul(&x120, &x120, &x60); + secp256k1_scalar_mul(&x112, &x112, &x56); - secp256k1_scalar_sqr(&x127, &x120); - for (i = 0; i < 6; i++) { - secp256k1_scalar_sqr(&x127, &x127); + secp256k1_scalar_sqr(&x126, &x112); + for (i = 0; i < 13; i++) { + secp256k1_scalar_sqr(&x126, &x126); } - secp256k1_scalar_mul(&x127, &x127, &x7); + secp256k1_scalar_mul(&x126, &x126, &x14); - /* Then accumulate the final result (t starts at x127). */ - t = &x127; - for (i = 0; i < 2; i++) { /* 0 */ + /* Then accumulate the final result (t starts at x126). */ + t = &x126; + for (i = 0; i < 3; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ + for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 4; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ + for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 3; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u11); /* 1011 */ + for (i = 0; i < 4; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ + secp256k1_scalar_mul(t, t, &u11); /* 1011 */ for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } @@ -156,38 +147,26 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x3); /* 111 */ - for (i = 0; i < 4; i++) { /* 00 */ + for (i = 0; i < 6; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 4; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ + for (i = 0; i < 3; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &x3); /* 111 */ for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x4); /* 1111 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) { /* 00 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 4; i++) { /* 000 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u9); /* 1001 */ + for (i = 0; i < 6; i++) { /* 000 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &u5); /* 101 */ for (i = 0; i < 10; i++) { /* 0000000 */ secp256k1_scalar_sqr(t, t); } @@ -200,50 +179,34 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x8); /* 11111111 */ - for (i = 0; i < 2; i++) { /* 0 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) { /* 00 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 3; i++) { /* 00 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, x); /* 1 */ for (i = 0; i < 5; i++) { /* 0 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x4); /* 1111 */ - for (i = 0; i < 2; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u9); /* 1001 */ + for (i = 0; i < 6; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 5; i++) { /* 000 */ + secp256k1_scalar_mul(t, t, &u11); /* 1011 */ + for (i = 0; i < 4; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 4; i++) { /* 00 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 5; i++) { secp256k1_scalar_sqr(t, t); } secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 2; i++) { /* 0 */ + for (i = 0; i < 6; i++) { /* 00 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ - for (i = 0; i < 8; i++) { /* 000000 */ - secp256k1_scalar_sqr(t, t); - } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 3; i++) { /* 0 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 10; i++) { /* 000000 */ secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, &x2); /* 11 */ - for (i = 0; i < 3; i++) { /* 00 */ + secp256k1_scalar_mul(t, t, &u13); /* 1101 */ + for (i = 0; i < 4; i++) { secp256k1_scalar_sqr(t, t); } - secp256k1_scalar_mul(t, t, x); /* 1 */ + secp256k1_scalar_mul(t, t, &u9); /* 1001 */ for (i = 0; i < 6; i++) { /* 00000 */ secp256k1_scalar_sqr(t, t); } diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index fb8b882faa..4f8c01655b 100755..100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -424,6 +424,33 @@ int secp256k1_ec_pubkey_create(const secp256k1_context* ctx, secp256k1_pubkey *p return ret; } +int secp256k1_ec_privkey_negate(const secp256k1_context* ctx, unsigned char *seckey) { + secp256k1_scalar sec; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(seckey != NULL); + + secp256k1_scalar_set_b32(&sec, seckey, NULL); + secp256k1_scalar_negate(&sec, &sec); + secp256k1_scalar_get_b32(seckey, &sec); + + return 1; +} + +int secp256k1_ec_pubkey_negate(const secp256k1_context* ctx, secp256k1_pubkey *pubkey) { + int ret = 0; + secp256k1_ge p; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(pubkey != NULL); + + ret = secp256k1_pubkey_load(ctx, &p, pubkey); + memset(pubkey, 0, sizeof(*pubkey)); + if (ret) { + secp256k1_ge_neg(&p, &p); + secp256k1_pubkey_save(pubkey, &p); + } + return ret; +} + int secp256k1_ec_privkey_tweak_add(const secp256k1_context* ctx, unsigned char *seckey, const unsigned char *tweak) { secp256k1_scalar term; secp256k1_scalar sec; @@ -552,10 +579,6 @@ int secp256k1_ec_pubkey_combine(const secp256k1_context* ctx, secp256k1_pubkey * # include "modules/ecdh/main_impl.h" #endif -#ifdef ENABLE_MODULE_SCHNORR -# include "modules/schnorr/main_impl.h" -#endif - #ifdef ENABLE_MODULE_RECOVERY # include "modules/recovery/main_impl.h" #endif diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index 9ae7d30281..3d9bd5ebb4 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -10,6 +10,7 @@ #include <stdio.h> #include <stdlib.h> +#include <string.h> #include <time.h> @@ -135,6 +136,7 @@ void random_scalar_order(secp256k1_scalar *num) { void run_context_tests(void) { secp256k1_pubkey pubkey; + secp256k1_pubkey zero_pubkey; secp256k1_ecdsa_signature sig; unsigned char ctmp[32]; int32_t ecount; @@ -149,6 +151,8 @@ void run_context_tests(void) { secp256k1_scalar msg, key, nonce; secp256k1_scalar sigr, sigs; + memset(&zero_pubkey, 0, sizeof(zero_pubkey)); + ecount = 0; ecount2 = 10; secp256k1_context_set_illegal_callback(vrfy, counting_illegal_callback_fn, &ecount); @@ -201,12 +205,20 @@ void run_context_tests(void) { CHECK(ecount == 2); CHECK(secp256k1_ec_pubkey_tweak_mul(sign, &pubkey, ctmp) == 0); CHECK(ecount2 == 13); - CHECK(secp256k1_ec_pubkey_tweak_mul(vrfy, &pubkey, ctmp) == 1); + CHECK(secp256k1_ec_pubkey_negate(vrfy, &pubkey) == 1); CHECK(ecount == 2); - CHECK(secp256k1_context_randomize(vrfy, ctmp) == 0); + CHECK(secp256k1_ec_pubkey_negate(sign, &pubkey) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_ec_pubkey_negate(sign, NULL) == 0); + CHECK(ecount2 == 14); + CHECK(secp256k1_ec_pubkey_negate(vrfy, &zero_pubkey) == 0); CHECK(ecount == 3); + CHECK(secp256k1_ec_pubkey_tweak_mul(vrfy, &pubkey, ctmp) == 1); + CHECK(ecount == 3); + CHECK(secp256k1_context_randomize(vrfy, ctmp) == 0); + CHECK(ecount == 4); CHECK(secp256k1_context_randomize(sign, NULL) == 1); - CHECK(ecount2 == 13); + CHECK(ecount2 == 14); secp256k1_context_set_illegal_callback(vrfy, NULL, NULL); secp256k1_context_set_illegal_callback(sign, NULL, NULL); @@ -1879,9 +1891,9 @@ void test_ge(void) { * * When the endomorphism code is compiled in, p5 = lambda*p1 and p6 = lambda^2*p1 are added as well. */ - secp256k1_ge *ge = (secp256k1_ge *)malloc(sizeof(secp256k1_ge) * (1 + 4 * runs)); - secp256k1_gej *gej = (secp256k1_gej *)malloc(sizeof(secp256k1_gej) * (1 + 4 * runs)); - secp256k1_fe *zinv = (secp256k1_fe *)malloc(sizeof(secp256k1_fe) * (1 + 4 * runs)); + secp256k1_ge *ge = (secp256k1_ge *)checked_malloc(&ctx->error_callback, sizeof(secp256k1_ge) * (1 + 4 * runs)); + secp256k1_gej *gej = (secp256k1_gej *)checked_malloc(&ctx->error_callback, sizeof(secp256k1_gej) * (1 + 4 * runs)); + secp256k1_fe *zinv = (secp256k1_fe *)checked_malloc(&ctx->error_callback, sizeof(secp256k1_fe) * (1 + 4 * runs)); secp256k1_fe zf; secp256k1_fe zfi2, zfi3; @@ -1919,7 +1931,7 @@ void test_ge(void) { /* Compute z inverses. */ { - secp256k1_fe *zs = malloc(sizeof(secp256k1_fe) * (1 + 4 * runs)); + secp256k1_fe *zs = checked_malloc(&ctx->error_callback, sizeof(secp256k1_fe) * (1 + 4 * runs)); for (i = 0; i < 4 * runs + 1; i++) { if (i == 0) { /* The point at infinity does not have a meaningful z inverse. Any should do. */ @@ -2020,7 +2032,7 @@ void test_ge(void) { /* Test adding all points together in random order equals infinity. */ { secp256k1_gej sum = SECP256K1_GEJ_CONST_INFINITY; - secp256k1_gej *gej_shuffled = (secp256k1_gej *)malloc((4 * runs + 1) * sizeof(secp256k1_gej)); + secp256k1_gej *gej_shuffled = (secp256k1_gej *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_gej)); for (i = 0; i < 4 * runs + 1; i++) { gej_shuffled[i] = gej[i]; } @@ -2041,9 +2053,9 @@ void test_ge(void) { /* Test batch gej -> ge conversion with and without known z ratios. */ { - secp256k1_fe *zr = (secp256k1_fe *)malloc((4 * runs + 1) * sizeof(secp256k1_fe)); - secp256k1_ge *ge_set_table = (secp256k1_ge *)malloc((4 * runs + 1) * sizeof(secp256k1_ge)); - secp256k1_ge *ge_set_all = (secp256k1_ge *)malloc((4 * runs + 1) * sizeof(secp256k1_ge)); + secp256k1_fe *zr = (secp256k1_fe *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_fe)); + secp256k1_ge *ge_set_table = (secp256k1_ge *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_ge)); + secp256k1_ge *ge_set_all = (secp256k1_ge *)checked_malloc(&ctx->error_callback, (4 * runs + 1) * sizeof(secp256k1_ge)); for (i = 0; i < 4 * runs + 1; i++) { /* Compute gej[i + 1].z / gez[i].z (with gej[n].z taken to be 1). */ if (i < 4 * runs) { @@ -3436,6 +3448,7 @@ void test_ecdsa_end_to_end(void) { unsigned char pubkeyc[65]; size_t pubkeyclen = 65; secp256k1_pubkey pubkey; + secp256k1_pubkey pubkey_tmp; unsigned char seckey[300]; size_t seckeylen = 300; @@ -3457,6 +3470,13 @@ void test_ecdsa_end_to_end(void) { memset(&pubkey, 0, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(ctx, &pubkey, pubkeyc, pubkeyclen) == 1); + /* Verify negation changes the key and changes it back */ + memcpy(&pubkey_tmp, &pubkey, sizeof(pubkey)); + CHECK(secp256k1_ec_pubkey_negate(ctx, &pubkey_tmp) == 1); + CHECK(memcmp(&pubkey_tmp, &pubkey, sizeof(pubkey)) != 0); + CHECK(secp256k1_ec_pubkey_negate(ctx, &pubkey_tmp) == 1); + CHECK(memcmp(&pubkey_tmp, &pubkey, sizeof(pubkey)) == 0); + /* Verify private key import and export. */ CHECK(ec_privkey_export_der(ctx, seckey, &seckeylen, privkey, secp256k1_rand_bits(1) == 1)); CHECK(ec_privkey_import_der(ctx, privkey2, seckey, seckeylen) == 1); @@ -4383,10 +4403,6 @@ void run_ecdsa_openssl(void) { # include "modules/ecdh/tests_impl.h" #endif -#ifdef ENABLE_MODULE_SCHNORR -# include "modules/schnorr/tests_impl.h" -#endif - #ifdef ENABLE_MODULE_RECOVERY # include "modules/recovery/tests_impl.h" #endif @@ -4504,11 +4520,6 @@ int main(int argc, char **argv) { run_ecdsa_openssl(); #endif -#ifdef ENABLE_MODULE_SCHNORR - /* Schnorr tests */ - run_schnorr_tests(); -#endif - #ifdef ENABLE_MODULE_RECOVERY /* ECDSA pubkey recovery tests */ run_recovery_tests(); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c index bda6ee475c..b040bb0733 100644 --- a/src/secp256k1/src/tests_exhaustive.c +++ b/src/secp256k1/src/tests_exhaustive.c @@ -26,6 +26,11 @@ #include "secp256k1.c" #include "testrand_impl.h" +#ifdef ENABLE_MODULE_RECOVERY +#include "src/modules/recovery/main_impl.h" +#include "include/secp256k1_recovery.h" +#endif + /** stolen from tests.c */ void ge_equals_ge(const secp256k1_ge *a, const secp256k1_ge *b) { CHECK(a->infinity == b->infinity); @@ -77,7 +82,7 @@ int secp256k1_nonce_function_smallint(unsigned char *nonce32, const unsigned cha * function with an increased `attempt`. So if attempt > 0 this means we * need to change the nonce to avoid an infinite loop. */ if (attempt > 0) { - (*idata)++; + *idata = (*idata + 1) % EXHAUSTIVE_TEST_ORDER; } secp256k1_scalar_set_int(&s, *idata); secp256k1_scalar_get_b32(nonce32, &s); @@ -244,6 +249,7 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou for (i = 1; i < order; i++) { /* message */ for (j = 1; j < order; j++) { /* key */ for (k = 1; k < order; k++) { /* nonce */ + const int starting_k = k; secp256k1_ecdsa_signature sig; secp256k1_scalar sk, msg, r, s, expected_r; unsigned char sk32[32], msg32[32]; @@ -262,6 +268,11 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou CHECK(r == expected_r); CHECK((k * s) % order == (i + r * j) % order || (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + + /* Overflow means we've tried every possible nonce */ + if (k < starting_k) { + break; + } } } } @@ -276,6 +287,130 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou */ } +#ifdef ENABLE_MODULE_RECOVERY +void test_exhaustive_recovery_sign(const secp256k1_context *ctx, const secp256k1_ge *group, int order) { + int i, j, k; + + /* Loop */ + for (i = 1; i < order; i++) { /* message */ + for (j = 1; j < order; j++) { /* key */ + for (k = 1; k < order; k++) { /* nonce */ + const int starting_k = k; + secp256k1_fe r_dot_y_normalized; + secp256k1_ecdsa_recoverable_signature rsig; + secp256k1_ecdsa_signature sig; + secp256k1_scalar sk, msg, r, s, expected_r; + unsigned char sk32[32], msg32[32]; + int expected_recid; + int recid; + secp256k1_scalar_set_int(&msg, i); + secp256k1_scalar_set_int(&sk, j); + secp256k1_scalar_get_b32(sk32, &sk); + secp256k1_scalar_get_b32(msg32, &msg); + + secp256k1_ecdsa_sign_recoverable(ctx, &rsig, msg32, sk32, secp256k1_nonce_function_smallint, &k); + + /* Check directly */ + secp256k1_ecdsa_recoverable_signature_load(ctx, &r, &s, &recid, &rsig); + r_from_k(&expected_r, group, k); + CHECK(r == expected_r); + CHECK((k * s) % order == (i + r * j) % order || + (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + /* In computing the recid, there is an overflow condition that is disabled in + * scalar_low_impl.h `secp256k1_scalar_set_b32` because almost every r.y value + * will exceed the group order, and our signing code always holds out for r + * values that don't overflow, so with a proper overflow check the tests would + * loop indefinitely. */ + r_dot_y_normalized = group[k].y; + secp256k1_fe_normalize(&r_dot_y_normalized); + /* Also the recovery id is flipped depending if we hit the low-s branch */ + if ((k * s) % order == (i + r * j) % order) { + expected_recid = secp256k1_fe_is_odd(&r_dot_y_normalized) ? 1 : 0; + } else { + expected_recid = secp256k1_fe_is_odd(&r_dot_y_normalized) ? 0 : 1; + } + CHECK(recid == expected_recid); + + /* Convert to a standard sig then check */ + secp256k1_ecdsa_recoverable_signature_convert(ctx, &sig, &rsig); + secp256k1_ecdsa_signature_load(ctx, &r, &s, &sig); + /* Note that we compute expected_r *after* signing -- this is important + * because our nonce-computing function function might change k during + * signing. */ + r_from_k(&expected_r, group, k); + CHECK(r == expected_r); + CHECK((k * s) % order == (i + r * j) % order || + (k * (EXHAUSTIVE_TEST_ORDER - s)) % order == (i + r * j) % order); + + /* Overflow means we've tried every possible nonce */ + if (k < starting_k) { + break; + } + } + } + } +} + +void test_exhaustive_recovery_verify(const secp256k1_context *ctx, const secp256k1_ge *group, int order) { + /* This is essentially a copy of test_exhaustive_verify, with recovery added */ + int s, r, msg, key; + for (s = 1; s < order; s++) { + for (r = 1; r < order; r++) { + for (msg = 1; msg < order; msg++) { + for (key = 1; key < order; key++) { + secp256k1_ge nonconst_ge; + secp256k1_ecdsa_recoverable_signature rsig; + secp256k1_ecdsa_signature sig; + secp256k1_pubkey pk; + secp256k1_scalar sk_s, msg_s, r_s, s_s; + secp256k1_scalar s_times_k_s, msg_plus_r_times_sk_s; + int recid = 0; + int k, should_verify; + unsigned char msg32[32]; + + secp256k1_scalar_set_int(&s_s, s); + secp256k1_scalar_set_int(&r_s, r); + secp256k1_scalar_set_int(&msg_s, msg); + secp256k1_scalar_set_int(&sk_s, key); + secp256k1_scalar_get_b32(msg32, &msg_s); + + /* Verify by hand */ + /* Run through every k value that gives us this r and check that *one* works. + * Note there could be none, there could be multiple, ECDSA is weird. */ + should_verify = 0; + for (k = 0; k < order; k++) { + secp256k1_scalar check_x_s; + r_from_k(&check_x_s, group, k); + if (r_s == check_x_s) { + secp256k1_scalar_set_int(&s_times_k_s, k); + secp256k1_scalar_mul(&s_times_k_s, &s_times_k_s, &s_s); + secp256k1_scalar_mul(&msg_plus_r_times_sk_s, &r_s, &sk_s); + secp256k1_scalar_add(&msg_plus_r_times_sk_s, &msg_plus_r_times_sk_s, &msg_s); + should_verify |= secp256k1_scalar_eq(&s_times_k_s, &msg_plus_r_times_sk_s); + } + } + /* nb we have a "high s" rule */ + should_verify &= !secp256k1_scalar_is_high(&s_s); + + /* We would like to try recovering the pubkey and checking that it matches, + * but pubkey recovery is impossible in the exhaustive tests (the reason + * being that there are 12 nonzero r values, 12 nonzero points, and no + * overlap between the sets, so there are no valid signatures). */ + + /* Verify by converting to a standard signature and calling verify */ + secp256k1_ecdsa_recoverable_signature_save(&rsig, &r_s, &s_s, recid); + secp256k1_ecdsa_recoverable_signature_convert(ctx, &sig, &rsig); + memcpy(&nonconst_ge, &group[sk_s], sizeof(nonconst_ge)); + secp256k1_pubkey_save(&pk, &nonconst_ge); + CHECK(should_verify == + secp256k1_ecdsa_verify(ctx, &sig, msg32, &pk)); + } + } + } + } +} +#endif + int main(void) { int i; secp256k1_gej groupj[EXHAUSTIVE_TEST_ORDER]; @@ -324,6 +459,12 @@ int main(void) { test_exhaustive_sign(ctx, group, EXHAUSTIVE_TEST_ORDER); test_exhaustive_verify(ctx, group, EXHAUSTIVE_TEST_ORDER); +#ifdef ENABLE_MODULE_RECOVERY + test_exhaustive_recovery_sign(ctx, group, EXHAUSTIVE_TEST_ORDER); + test_exhaustive_recovery_verify(ctx, group, EXHAUSTIVE_TEST_ORDER); +#endif + + secp256k1_context_destroy(ctx); return 0; } diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index 4eef4ded47..4092a86c91 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -57,7 +57,10 @@ static SECP256K1_INLINE void secp256k1_callback_call(const secp256k1_callback * #endif /* Like assert(), but when VERIFY is defined, and side-effect safe. */ -#ifdef VERIFY +#if defined(COVERAGE) +#define VERIFY_CHECK(check) +#define VERIFY_SETUP(stmt) +#elif defined(VERIFY) #define VERIFY_CHECK CHECK #define VERIFY_SETUP(stmt) do { stmt; } while(0) #else diff --git a/src/serialize.h b/src/serialize.h index e4d72d2348..e82ddf2c5a 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -336,11 +336,18 @@ I ReadVarInt(Stream& is) I n = 0; while(true) { unsigned char chData = ser_readdata8(is); + if (n > (std::numeric_limits<I>::max() >> 7)) { + throw std::ios_base::failure("ReadVarInt(): size too large"); + } n = (n << 7) | (chData & 0x7F); - if (chData & 0x80) + if (chData & 0x80) { + if (n == std::numeric_limits<I>::max()) { + throw std::ios_base::failure("ReadVarInt(): size too large"); + } n++; - else + } else { return n; + } } } diff --git a/src/streams.h b/src/streams.h index 1387b9cf54..8dc5a19ead 100644 --- a/src/streams.h +++ b/src/streams.h @@ -248,7 +248,8 @@ public: void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last) { - assert(last - first >= 0); + if (last == first) return; + assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room @@ -261,7 +262,8 @@ public: void insert(iterator it, const char* first, const char* last) { - assert(last - first >= 0); + if (last == first) return; + assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room @@ -339,6 +341,8 @@ public: void read(char* pch, size_t nSize) { + if (nSize == 0) return; + // Read from the beginning of the buffer unsigned int nReadPosNext = nReadPos + nSize; if (nReadPosNext >= vch.size()) diff --git a/src/support/cleanse.h b/src/support/cleanse.h index 3e02aa8fd1..f020216c73 100644 --- a/src/support/cleanse.h +++ b/src/support/cleanse.h @@ -8,6 +8,7 @@ #include <stdlib.h> +// Attempt to overwrite data in the specified memory span. void memory_cleanse(void *ptr, size_t len); #endif // BITCOIN_SUPPORT_CLEANSE_H diff --git a/src/test/DoS_tests.cpp b/src/test/DoS_tests.cpp index c62e6ae838..e6b45a3b5e 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/DoS_tests.cpp @@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) connman->ClearBanned(); CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, "", true); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 0, 0, CAddress(), "", true); dummyNode1.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode1, *connman); dummyNode1.nVersion = 1; @@ -62,7 +62,7 @@ BOOST_AUTO_TEST_CASE(DoS_banning) BOOST_CHECK(!connman->IsBanned(ip(0xa0b0c001|0x0000ff00))); // Different IP, not banned CAddress addr2(ip(0xa0b0c002), NODE_NONE); - CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, "", true); + CNode dummyNode2(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr2, 1, 1, CAddress(), "", true); dummyNode2.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode2, *connman); dummyNode2.nVersion = 1; @@ -83,7 +83,7 @@ BOOST_AUTO_TEST_CASE(DoS_banscore) connman->ClearBanned(); ForceSetArg("-banscore", "111"); // because 11 is my favorite number CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, "", true); + CNode dummyNode1(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr1, 3, 1, CAddress(), "", true); dummyNode1.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode1, *connman); dummyNode1.nVersion = 1; @@ -109,7 +109,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, "", true); + CNode dummyNode(id++, NODE_NETWORK, 0, INVALID_SOCKET, addr, 4, 4, CAddress(), "", true); dummyNode.SetSendVersion(PROTOCOL_VERSION); GetNodeSignals().InitializeNode(&dummyNode, *connman); dummyNode.nVersion = 1; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 5d1c5b78d1..dc5372a070 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -84,36 +84,47 @@ BOOST_AUTO_TEST_CASE(addrman_simple) CNetAddr source = ResolveIP("252.2.2.2"); - // Test 1: Does Addrman respond correctly when empty. - BOOST_CHECK(addrman.size() == 0); + // Test: Does Addrman respond correctly when empty. + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo addr_null = addrman.Select(); - BOOST_CHECK(addr_null.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_null.ToString(), "[::]:0"); - // Test 2: Does Addrman::Add work as expected. + // Test: Does Addrman::Add work as expected. CService addr1 = ResolveService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK(addrman.Add(CAddress(addr1, NODE_NONE), source)); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret1 = addrman.Select(); - BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); - // Test 3: Does IP address deduplication work correctly. + // Test: Does IP address deduplication work correctly. // Expected dup IP should not be added. CService addr1_dup = ResolveService("250.1.1.1", 8333); - addrman.Add(CAddress(addr1_dup, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK(!addrman.Add(CAddress(addr1_dup, NODE_NONE), source)); + BOOST_CHECK_EQUAL(addrman.size(), 1); + + // Test: New table has one addr and we add a diff addr we should + // have at least one addr. + // Note that addrman's size cannot be tested reliably after insertion, as + // hash collisions may occur. But we can always be sure of at least one + // success. - // Test 5: New table has one addr and we add a diff addr we should - // have two addrs. CService addr2 = ResolveService("250.1.1.2", 8333); - addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 2); + BOOST_CHECK(addrman.Add(CAddress(addr2, NODE_NONE), source)); + BOOST_CHECK(addrman.size() >= 1); - // Test 6: AddrMan::Clear() should empty the new table. + // Test: AddrMan::Clear() should empty the new table. addrman.Clear(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo addr_null2 = addrman.Select(); - BOOST_CHECK(addr_null2.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_null2.ToString(), "[::]:0"); + + // Test: AddrMan::Add multiple addresses works as expected + std::vector<CAddress> vAddr; + vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); + vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); + BOOST_CHECK(addrman.Add(vAddr, source)); + BOOST_CHECK(addrman.size() >= 1); } BOOST_AUTO_TEST_CASE(addrman_ports) @@ -125,26 +136,26 @@ BOOST_AUTO_TEST_CASE(addrman_ports) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); // Test 7; Addr with same IP but diff port does not replace existing addr. CService addr1 = ResolveService("250.1.1.1", 8333); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CService addr1_port = ResolveService("250.1.1.1", 8334); addrman.Add(CAddress(addr1_port, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret2 = addrman.Select(); - BOOST_CHECK(addr_ret2.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret2.ToString(), "250.1.1.1:8333"); - // Test 8: Add same IP but diff port to tried table, it doesn't get added. + // Test: Add same IP but diff port to tried table, it doesn't get added. // Perhaps this is not ideal behavior but it is the current behavior. addrman.Good(CAddress(addr1_port, NODE_NONE)); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); bool newOnly = true; CAddrInfo addr_ret3 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret3.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); } @@ -157,25 +168,25 @@ BOOST_AUTO_TEST_CASE(addrman_select) CNetAddr source = ResolveIP("252.2.2.2"); - // Test 9: Select from new with 1 addr in new. + // Test: Select from new with 1 addr in new. CService addr1 = ResolveService("250.1.1.1", 8333); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); bool newOnly = true; CAddrInfo addr_ret1 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret1.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); - // Test 10: move addr to tried, select from new expected nothing returned. + // Test: move addr to tried, select from new expected nothing returned. addrman.Good(CAddress(addr1, NODE_NONE)); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); CAddrInfo addr_ret2 = addrman.Select(newOnly); - BOOST_CHECK(addr_ret2.ToString() == "[::]:0"); + BOOST_CHECK_EQUAL(addr_ret2.ToString(), "[::]:0"); CAddrInfo addr_ret3 = addrman.Select(); - BOOST_CHECK(addr_ret3.ToString() == "250.1.1.1:8333"); + BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); - BOOST_CHECK(addrman.size() == 1); + BOOST_CHECK_EQUAL(addrman.size(), 1); // Add three addresses to new table. @@ -199,14 +210,15 @@ BOOST_AUTO_TEST_CASE(addrman_select) addrman.Add(CAddress(addr7, NODE_NONE), ResolveService("250.1.1.3", 8333)); addrman.Good(CAddress(addr7, NODE_NONE)); - // Test 11: 6 addrs + 1 addr from last test = 7. - BOOST_CHECK(addrman.size() == 7); + // Test: 6 addrs + 1 addr from last test = 7. + BOOST_CHECK_EQUAL(addrman.size(), 7); - // Test 12: Select pulls from new and tried regardless of port number. - BOOST_CHECK(addrman.Select().ToString() == "250.4.6.6:8333"); - BOOST_CHECK(addrman.Select().ToString() == "250.3.2.2:9999"); - BOOST_CHECK(addrman.Select().ToString() == "250.3.3.3:9999"); - BOOST_CHECK(addrman.Select().ToString() == "250.4.4.4:8333"); + // Test: Select pulls from new and tried regardless of port number. + std::set<uint16_t> ports; + for (int i = 0; i < 20; ++i) { + ports.insert(addrman.Select().GetPort()); + } + BOOST_CHECK_EQUAL(ports.size(), 3); } BOOST_AUTO_TEST_CASE(addrman_new_collisions) @@ -218,24 +230,24 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); for (unsigned int i = 1; i < 18; i++) { CService addr = ResolveService("250.1.1." + boost::to_string(i)); addrman.Add(CAddress(addr, NODE_NONE), source); - //Test 13: No collision in new table yet. - BOOST_CHECK(addrman.size() == i); + //Test: No collision in new table yet. + BOOST_CHECK_EQUAL(addrman.size(), i); } - //Test 14: new table collision! + //Test: new table collision! CService addr1 = ResolveService("250.1.1.18"); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 17); + BOOST_CHECK_EQUAL(addrman.size(), 17); CService addr2 = ResolveService("250.1.1.19"); addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 18); + BOOST_CHECK_EQUAL(addrman.size(), 18); } BOOST_AUTO_TEST_CASE(addrman_tried_collisions) @@ -247,25 +259,25 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); for (unsigned int i = 1; i < 80; i++) { CService addr = ResolveService("250.1.1." + boost::to_string(i)); addrman.Add(CAddress(addr, NODE_NONE), source); addrman.Good(CAddress(addr, NODE_NONE)); - //Test 15: No collision in tried table yet. + //Test: No collision in tried table yet. BOOST_CHECK_EQUAL(addrman.size(), i); } - //Test 16: tried table collision! + //Test: tried table collision! CService addr1 = ResolveService("250.1.1.80"); addrman.Add(CAddress(addr1, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 79); + BOOST_CHECK_EQUAL(addrman.size(), 79); CService addr2 = ResolveService("250.1.1.81"); addrman.Add(CAddress(addr2, NODE_NONE), source); - BOOST_CHECK(addrman.size() == 80); + BOOST_CHECK_EQUAL(addrman.size(), 80); } BOOST_AUTO_TEST_CASE(addrman_find) @@ -275,7 +287,7 @@ BOOST_AUTO_TEST_CASE(addrman_find) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); @@ -288,23 +300,20 @@ BOOST_AUTO_TEST_CASE(addrman_find) addrman.Add(addr2, source2); addrman.Add(addr3, source1); - // Test 17: ensure Find returns an IP matching what we searched on. + // Test: ensure Find returns an IP matching what we searched on. CAddrInfo* info1 = addrman.Find(addr1); - BOOST_CHECK(info1); - if (info1) - BOOST_CHECK(info1->ToString() == "250.1.2.1:8333"); + BOOST_REQUIRE(info1); + BOOST_CHECK_EQUAL(info1->ToString(), "250.1.2.1:8333"); // Test 18; Find does not discriminate by port number. CAddrInfo* info2 = addrman.Find(addr2); - BOOST_CHECK(info2); - if (info2 && info1) - BOOST_CHECK(info2->ToString() == info1->ToString()); + BOOST_REQUIRE(info2); + BOOST_CHECK_EQUAL(info2->ToString(), info1->ToString()); - // Test 19: Find returns another IP matching what we searched on. + // Test: Find returns another IP matching what we searched on. CAddrInfo* info3 = addrman.Find(addr3); - BOOST_CHECK(info3); - if (info3) - BOOST_CHECK(info3->ToString() == "251.255.2.1:8333"); + BOOST_REQUIRE(info3); + BOOST_CHECK_EQUAL(info3->ToString(), "251.255.2.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_create) @@ -314,7 +323,7 @@ BOOST_AUTO_TEST_CASE(addrman_create) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); @@ -322,11 +331,11 @@ BOOST_AUTO_TEST_CASE(addrman_create) int nId; CAddrInfo* pinfo = addrman.Create(addr1, source1, &nId); - // Test 20: The result should be the same as the input addr. - BOOST_CHECK(pinfo->ToString() == "250.1.2.1:8333"); + // Test: The result should be the same as the input addr. + BOOST_CHECK_EQUAL(pinfo->ToString(), "250.1.2.1:8333"); CAddrInfo* info2 = addrman.Find(addr1); - BOOST_CHECK(info2->ToString() == "250.1.2.1:8333"); + BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:8333"); } @@ -337,7 +346,7 @@ BOOST_AUTO_TEST_CASE(addrman_delete) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); @@ -345,10 +354,10 @@ BOOST_AUTO_TEST_CASE(addrman_delete) int nId; addrman.Create(addr1, source1, &nId); - // Test 21: Delete should actually delete the addr. - BOOST_CHECK(addrman.size() == 1); + // Test: Delete should actually delete the addr. + BOOST_CHECK_EQUAL(addrman.size(), 1); addrman.Delete(nId); - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); CAddrInfo* info2 = addrman.Find(addr1); BOOST_CHECK(info2 == NULL); } @@ -360,11 +369,11 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) // Set addrman addr placement to be deterministic. addrman.MakeDeterministic(); - // Test 22: Sanity check, GetAddr should never return anything if addrman + // Test: Sanity check, GetAddr should never return anything if addrman // is empty. - BOOST_CHECK(addrman.size() == 0); + BOOST_CHECK_EQUAL(addrman.size(), 0); std::vector<CAddress> vAddr1 = addrman.GetAddr(); - BOOST_CHECK(vAddr1.size() == 0); + BOOST_CHECK_EQUAL(vAddr1.size(), 0); CAddress addr1 = CAddress(ResolveService("250.250.2.1", 8333), NODE_NONE); addr1.nTime = GetAdjustedTime(); // Set time so isTerrible = false @@ -379,7 +388,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) CNetAddr source1 = ResolveIP("250.1.2.1"); CNetAddr source2 = ResolveIP("250.2.3.3"); - // Test 23: Ensure GetAddr works with new addresses. + // Test: Ensure GetAddr works with new addresses. addrman.Add(addr1, source1); addrman.Add(addr2, source2); addrman.Add(addr3, source1); @@ -387,21 +396,20 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) addrman.Add(addr5, source1); // GetAddr returns 23% of addresses, 23% of 5 is 1 rounded down. - BOOST_CHECK(addrman.GetAddr().size() == 1); + BOOST_CHECK_EQUAL(addrman.GetAddr().size(), 1); - // Test 24: Ensure GetAddr works with new and tried addresses. + // Test: Ensure GetAddr works with new and tried addresses. addrman.Good(CAddress(addr1, NODE_NONE)); addrman.Good(CAddress(addr2, NODE_NONE)); - BOOST_CHECK(addrman.GetAddr().size() == 1); + BOOST_CHECK_EQUAL(addrman.GetAddr().size(), 1); - // Test 25: Ensure GetAddr still returns 23% when addrman has many addrs. + // Test: Ensure GetAddr still returns 23% when addrman has many addrs. for (unsigned int i = 1; i < (8 * 256); i++) { int octet1 = i % 256; - int octet2 = (i / 256) % 256; - int octet3 = (i / (256 * 2)) % 256; - std::string strAddr = boost::to_string(octet1) + "." + boost::to_string(octet2) + "." + boost::to_string(octet3) + ".23"; + int octet2 = i >> 8 % 256; + std::string strAddr = boost::to_string(octet1) + "." + boost::to_string(octet2) + ".1.23"; CAddress addr = CAddress(ResolveService(strAddr), NODE_NONE); - + // Ensure that for all addrs in addrman, isTerrible == false. addr.nTime = GetAdjustedTime(); addrman.Add(addr, ResolveIP(strAddr)); @@ -411,10 +419,10 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) std::vector<CAddress> vAddr = addrman.GetAddr(); size_t percent23 = (addrman.size() * 23) / 100; - BOOST_CHECK(vAddr.size() == percent23); - BOOST_CHECK(vAddr.size() == 461); + BOOST_CHECK_EQUAL(vAddr.size(), percent23); + BOOST_CHECK_EQUAL(vAddr.size(), 461); // (Addrman.size() < number of addresses added) due to address collisions. - BOOST_CHECK(addrman.size() == 2007); + BOOST_CHECK_EQUAL(addrman.size(), 2006); } @@ -437,13 +445,13 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - BOOST_CHECK(info1.GetTriedBucket(nKey1) == 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1), 40); - // Test 26: Make sure key actually randomizes bucket placement. A fail on + // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. BOOST_CHECK(info1.GetTriedBucket(nKey1) != info1.GetTriedBucket(nKey2)); - // Test 27: Two addresses with same IP but different ports can map to + // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. CAddrInfo info2 = CAddrInfo(addr2, source1); @@ -458,9 +466,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) int bucket = infoi.GetTriedBucket(nKey1); buckets.insert(bucket); } - // Test 28: IP addresses in the same group (\16 prefix for IPv4) should + // Test: IP addresses in the same group (\16 prefix for IPv4) should // never get more than 8 buckets - BOOST_CHECK(buckets.size() == 8); + BOOST_CHECK_EQUAL(buckets.size(), 8); buckets.clear(); for (int j = 0; j < 255; j++) { @@ -470,9 +478,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) int bucket = infoj.GetTriedBucket(nKey1); buckets.insert(bucket); } - // Test 29: IP addresses in the different groups should map to more than + // Test: IP addresses in the different groups should map to more than // 8 buckets. - BOOST_CHECK(buckets.size() == 160); + BOOST_CHECK_EQUAL(buckets.size(), 160); } BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) @@ -492,16 +500,18 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - BOOST_CHECK(info1.GetNewBucket(nKey1) == 786); + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1), 786); - // Test 30: Make sure key actually randomizes bucket placement. A fail on + // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. BOOST_CHECK(info1.GetNewBucket(nKey1) != info1.GetNewBucket(nKey2)); - // Test 31: Ports should not effect bucket placement in the addr + // Test: Ports should not effect bucket placement in the addr CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetNewBucket(nKey1) == info2.GetNewBucket(nKey1)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), info2.GetNewBucket(nKey1)); std::set<int> buckets; for (int i = 0; i < 255; i++) { @@ -511,9 +521,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoi.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 32: IP addresses in the same group (\16 prefix for IPv4) should + // Test: IP addresses in the same group (\16 prefix for IPv4) should // always map to the same bucket. - BOOST_CHECK(buckets.size() == 1); + BOOST_CHECK_EQUAL(buckets.size(), 1); buckets.clear(); for (int j = 0; j < 4 * 255; j++) { @@ -524,7 +534,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 33: IP addresses in the same source groups should map to no more + // Test: IP addresses in the same source groups should map to no more // than 64 buckets. BOOST_CHECK(buckets.size() <= 64); @@ -536,7 +546,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) int bucket = infoj.GetNewBucket(nKey1); buckets.insert(bucket); } - // Test 34: IP addresses in the different source groups should map to more + // Test: IP addresses in the different source groups should map to more // than 64 buckets. BOOST_CHECK(buckets.size() > 64); } diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index fd6f88b366..952cf901f0 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -3,15 +3,23 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "amount.h" +#include "policy/feerate.h" #include "test/test_bitcoin.h" #include <boost/test/unit_test.hpp> BOOST_FIXTURE_TEST_SUITE(amount_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(MoneyRangeTest) +{ + BOOST_CHECK_EQUAL(MoneyRange(CAmount(-1)), false); + BOOST_CHECK_EQUAL(MoneyRange(MAX_MONEY + CAmount(1)), false); + BOOST_CHECK_EQUAL(MoneyRange(CAmount(1)), true); +} + BOOST_AUTO_TEST_CASE(GetFeeTest) { - CFeeRate feeRate; + CFeeRate feeRate, altFeeRate; feeRate = CFeeRate(0); // Must always return 0 @@ -53,6 +61,11 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) BOOST_CHECK_EQUAL(feeRate.GetFee(8), -1); // Special case: returns -1 instead of 0 BOOST_CHECK_EQUAL(feeRate.GetFee(9), -1); + // check alternate constructor + feeRate = CFeeRate(1000); + altFeeRate = CFeeRate(feeRate); + BOOST_CHECK_EQUAL(feeRate.GetFee(100), altFeeRate.GetFee(100)); + // Check full constructor // default value BOOST_CHECK(CFeeRate(CAmount(-1), 1000) == CFeeRate(-1)); @@ -68,4 +81,28 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) CFeeRate(MAX_MONEY, std::numeric_limits<size_t>::max() >> 1).GetFeePerK(); } +BOOST_AUTO_TEST_CASE(BinaryOperatorTest) +{ + CFeeRate a, b; + a = CFeeRate(1); + b = CFeeRate(2); + BOOST_CHECK(a < b); + BOOST_CHECK(b > a); + BOOST_CHECK(a == a); + BOOST_CHECK(a <= b); + BOOST_CHECK(a <= a); + BOOST_CHECK(b >= a); + BOOST_CHECK(b >= b); + // a should be 0.00000002 BTC/kB now + a += a; + BOOST_CHECK(a == b); +} + +BOOST_AUTO_TEST_CASE(ToStringTest) +{ + CFeeRate feeRate; + feeRate = CFeeRate(1); + BOOST_CHECK_EQUAL(feeRate.ToString(), "0.00000001 BTC/kB"); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 31ed1a50b9..0ed71b96fc 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -17,35 +17,44 @@ #include <boost/test/unit_test.hpp> -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out); +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out); void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight); namespace { +//! equality test +bool operator==(const Coin &a, const Coin &b) { + // Empty Coin objects are always equal. + if (a.IsSpent() && b.IsSpent()) return true; + return a.fCoinBase == b.fCoinBase && + a.nHeight == b.nHeight && + a.out == b.out; +} + class CCoinsViewTest : public CCoinsView { uint256 hashBestBlock_; - std::map<uint256, CCoins> map_; + std::map<COutPoint, Coin> map_; public: - bool GetCoins(const uint256& txid, CCoins& coins) const + bool GetCoin(const COutPoint& outpoint, Coin& coin) const { - std::map<uint256, CCoins>::const_iterator it = map_.find(txid); + std::map<COutPoint, Coin>::const_iterator it = map_.find(outpoint); if (it == map_.end()) { return false; } - coins = it->second; - if (coins.IsPruned() && insecure_rand() % 2 == 0) { + coin = it->second; + if (coin.IsSpent() && insecure_rand() % 2 == 0) { // Randomly return false in case of an empty entry. return false; } return true; } - bool HaveCoins(const uint256& txid) const + bool HaveCoin(const COutPoint& outpoint) const { - CCoins coins; - return GetCoins(txid, coins); + Coin coin; + return GetCoin(outpoint, coin); } uint256 GetBestBlock() const { return hashBestBlock_; } @@ -55,8 +64,8 @@ public: for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end(); ) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { // Same optimization used in CCoinsViewDB is to only write dirty entries. - map_[it->first] = it->second.coins; - if (it->second.coins.IsPruned() && insecure_rand() % 3 == 0) { + map_[it->first] = it->second.coin; + if (it->second.coin.IsSpent() && insecure_rand() % 3 == 0) { // Randomly delete empty entries on write. map_.erase(it->first); } @@ -78,9 +87,12 @@ public: { // Manually recompute the dynamic usage of the whole data, and compare it. size_t ret = memusage::DynamicUsage(cacheCoins); + size_t count = 0; for (CCoinsMap::iterator it = cacheCoins.begin(); it != cacheCoins.end(); it++) { - ret += it->second.coins.DynamicMemoryUsage(); + ret += it->second.coin.DynamicMemoryUsage(); + ++count; } + BOOST_CHECK_EQUAL(GetCacheSize(), count); BOOST_CHECK_EQUAL(DynamicMemoryUsage(), ret); } @@ -97,7 +109,7 @@ static const unsigned int NUM_SIMULATION_ITERATIONS = 40000; // This is a large randomized insert/remove simulation test on a variable-size // stack of caches on top of CCoinsViewTest. // -// It will randomly create/update/delete CCoins entries to a tip of caches, with +// It will randomly create/update/delete Coin entries to a tip of caches, with // txids picked from a limited list of random 256-bit hashes. Occasionally, a // new tip is added to the stack of caches, or the tip is flushed and removed. // @@ -109,13 +121,15 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) bool removed_all_caches = false; bool reached_4_caches = false; bool added_an_entry = false; + bool added_an_unspendable_entry = false; bool removed_an_entry = false; bool updated_an_entry = false; bool found_an_entry = false; bool missed_an_entry = false; + bool uncached_an_entry = false; // A simple map to track what we expect the cache stack to represent. - std::map<uint256, CCoins> result; + std::map<COutPoint, Coin> result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -133,36 +147,51 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Do a random modification. { uint256 txid = txids[insecure_rand() % txids.size()]; // txid we're going to modify in this iteration. - CCoins& coins = result[txid]; - CCoinsModifier entry = stack.back()->ModifyCoins(txid); - BOOST_CHECK(coins == *entry); - if (insecure_rand() % 5 == 0 || coins.IsPruned()) { - if (coins.IsPruned()) { - added_an_entry = true; + Coin& coin = result[COutPoint(txid, 0)]; + const Coin& entry = (insecure_rand() % 500 == 0) ? AccessByTxid(*stack.back(), txid) : stack.back()->AccessCoin(COutPoint(txid, 0)); + BOOST_CHECK(coin == entry); + + if (insecure_rand() % 5 == 0 || coin.IsSpent()) { + Coin newcoin; + newcoin.out.nValue = insecure_rand(); + newcoin.nHeight = 1; + if (insecure_rand() % 16 == 0 && coin.IsSpent()) { + newcoin.out.scriptPubKey.assign(1 + (insecure_rand() & 0x3F), OP_RETURN); + BOOST_CHECK(newcoin.out.scriptPubKey.IsUnspendable()); + added_an_unspendable_entry = true; } else { - updated_an_entry = true; + newcoin.out.scriptPubKey.assign(insecure_rand() & 0x3F, 0); // Random sizes so we can test memory usage accounting + (coin.IsSpent() ? added_an_entry : updated_an_entry) = true; + coin = newcoin; } - coins.nVersion = insecure_rand(); - coins.vout.resize(1); - coins.vout[0].nValue = insecure_rand(); - *entry = coins; + stack.back()->AddCoin(COutPoint(txid, 0), std::move(newcoin), !coin.IsSpent() || insecure_rand() & 1); } else { - coins.Clear(); - entry->Clear(); removed_an_entry = true; + coin.Clear(); + stack.back()->SpendCoin(COutPoint(txid, 0)); } } + // One every 10 iterations, remove a random entry from the cache + if (insecure_rand() % 10 == 0) { + COutPoint out(txids[insecure_rand() % txids.size()], 0); + int cacheid = insecure_rand() % stack.size(); + stack[cacheid]->Uncache(out); + uncached_an_entry |= !stack[cacheid]->HaveCoinInCache(out); + } + // Once every 1000 iterations and at the end, verify the full cache. if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map<uint256, CCoins>::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - found_an_entry = true; - } else { - BOOST_CHECK(it->second.IsPruned()); + for (auto it = result.begin(); it != result.end(); it++) { + bool have = stack.back()->HaveCoin(it->first); + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(have == !coin.IsSpent()); + BOOST_CHECK(coin == it->second); + if (coin.IsSpent()) { missed_an_entry = true; + } else { + BOOST_CHECK(stack.back()->HaveCoinInCache(it->first)); + found_an_entry = true; } } BOOST_FOREACH(const CCoinsViewCacheTest *test, stack) { @@ -211,25 +240,27 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) BOOST_CHECK(removed_all_caches); BOOST_CHECK(reached_4_caches); BOOST_CHECK(added_an_entry); + BOOST_CHECK(added_an_unspendable_entry); BOOST_CHECK(removed_an_entry); BOOST_CHECK(updated_an_entry); BOOST_CHECK(found_an_entry); BOOST_CHECK(missed_an_entry); + BOOST_CHECK(uncached_an_entry); } -typedef std::tuple<CTransaction,CTxUndo,CCoins> TxData; // Store of all necessary tx and undo data for next test -std::map<uint256, TxData> alltxs; - -TxData &FindRandomFrom(const std::set<uint256> &txidset) { - assert(txidset.size()); - std::set<uint256>::iterator txIt = txidset.lower_bound(GetRandHash()); - if (txIt == txidset.end()) { - txIt = txidset.begin(); +typedef std::map<COutPoint, std::tuple<CTransaction,CTxUndo,Coin>> UtxoData; +UtxoData utxoData; + +UtxoData::iterator FindRandomFrom(const std::set<COutPoint> &utxoSet) { + assert(utxoSet.size()); + auto utxoSetIt = utxoSet.lower_bound(COutPoint(GetRandHash(), 0)); + if (utxoSetIt == utxoSet.end()) { + utxoSetIt = utxoSet.begin(); } - std::map<uint256, TxData>::iterator txdit = alltxs.find(*txIt); - assert(txdit != alltxs.end()); - return txdit->second; + auto utxoDataIt = utxoData.find(*utxoSetIt); + assert(utxoDataIt != utxoData.end()); + return utxoDataIt; } @@ -242,7 +273,7 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) { bool spent_a_duplicate_coinbase = false; // A simple map to track what we expect the cache stack to represent. - std::map<uint256, CCoins> result; + std::map<COutPoint, Coin> result; // The cache stack. CCoinsViewTest base; // A CCoinsViewTest at the bottom. @@ -250,10 +281,10 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. // Track the txids we've used in various sets - std::set<uint256> coinbaseids; - std::set<uint256> disconnectedids; - std::set<uint256> duplicateids; - std::set<uint256> utxoset; + std::set<COutPoint> coinbase_coins; + std::set<COutPoint> disconnected_coins; + std::set<COutPoint> duplicate_coins; + std::set<COutPoint> utxoset; for (unsigned int i = 0; i < NUM_SIMULATION_ITERATIONS; i++) { uint32_t randiter = insecure_rand(); @@ -264,23 +295,24 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) tx.vin.resize(1); tx.vout.resize(1); tx.vout[0].nValue = i; //Keep txs unique unless intended to duplicate + tx.vout[0].scriptPubKey.assign(insecure_rand() & 0x3F, 0); // Random sizes so we can test memory usage accounting unsigned int height = insecure_rand(); - CCoins oldcoins; + Coin old_coin; // 2/20 times create a new coinbase - if (randiter % 20 < 2 || coinbaseids.size() < 10) { + if (randiter % 20 < 2 || coinbase_coins.size() < 10) { // 1/10 of those times create a duplicate coinbase - if (insecure_rand() % 10 == 0 && coinbaseids.size()) { - TxData &txd = FindRandomFrom(coinbaseids); + if (insecure_rand() % 10 == 0 && coinbase_coins.size()) { + auto utxod = FindRandomFrom(coinbase_coins); // Reuse the exact same coinbase - tx = std::get<0>(txd); + tx = std::get<0>(utxod->second); // shouldn't be available for reconnection if its been duplicated - disconnectedids.erase(tx.GetHash()); + disconnected_coins.erase(utxod->first); - duplicateids.insert(tx.GetHash()); + duplicate_coins.insert(utxod->first); } else { - coinbaseids.insert(tx.GetHash()); + coinbase_coins.insert(COutPoint(tx.GetHash(), 0)); } assert(CTransaction(tx).IsCoinBase()); } @@ -288,114 +320,118 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) // 17/20 times reconnect previous or add a regular tx else { - uint256 prevouthash; + COutPoint prevout; // 1/20 times reconnect a previously disconnected tx - if (randiter % 20 == 2 && disconnectedids.size()) { - TxData &txd = FindRandomFrom(disconnectedids); - tx = std::get<0>(txd); - prevouthash = tx.vin[0].prevout.hash; - if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevouthash)) { - disconnectedids.erase(tx.GetHash()); + if (randiter % 20 == 2 && disconnected_coins.size()) { + auto utxod = FindRandomFrom(disconnected_coins); + tx = std::get<0>(utxod->second); + prevout = tx.vin[0].prevout; + if (!CTransaction(tx).IsCoinBase() && !utxoset.count(prevout)) { + disconnected_coins.erase(utxod->first); continue; } // If this tx is already IN the UTXO, then it must be a coinbase, and it must be a duplicate - if (utxoset.count(tx.GetHash())) { + if (utxoset.count(utxod->first)) { assert(CTransaction(tx).IsCoinBase()); - assert(duplicateids.count(tx.GetHash())); + assert(duplicate_coins.count(utxod->first)); } - disconnectedids.erase(tx.GetHash()); + disconnected_coins.erase(utxod->first); } // 16/20 times create a regular tx else { - TxData &txd = FindRandomFrom(utxoset); - prevouthash = std::get<0>(txd).GetHash(); + auto utxod = FindRandomFrom(utxoset); + prevout = utxod->first; // Construct the tx to spend the coins of prevouthash - tx.vin[0].prevout.hash = prevouthash; - tx.vin[0].prevout.n = 0; + tx.vin[0].prevout = prevout; assert(!CTransaction(tx).IsCoinBase()); } // In this simple test coins only have two states, spent or unspent, save the unspent state to restore - oldcoins = result[prevouthash]; + old_coin = result[prevout]; // Update the expected result of prevouthash to know these coins are spent - result[prevouthash].Clear(); + result[prevout].Clear(); - utxoset.erase(prevouthash); + utxoset.erase(prevout); // The test is designed to ensure spending a duplicate coinbase will work properly // if that ever happens and not resurrect the previously overwritten coinbase - if (duplicateids.count(prevouthash)) + if (duplicate_coins.count(prevout)) { spent_a_duplicate_coinbase = true; + } } // Update the expected result to know about the new output coins - result[tx.GetHash()].FromTx(tx, height); + assert(tx.vout.size() == 1); + const COutPoint outpoint(tx.GetHash(), 0); + result[outpoint] = Coin(tx.vout[0], height, CTransaction(tx).IsCoinBase()); // Call UpdateCoins on the top cache CTxUndo undo; UpdateCoins(tx, *(stack.back()), undo, height); // Update the utxo set for future spends - utxoset.insert(tx.GetHash()); + utxoset.insert(outpoint); // Track this tx and undo info to use later - alltxs.insert(std::make_pair(tx.GetHash(),std::make_tuple(tx,undo,oldcoins))); - } - - //1/20 times undo a previous transaction - else if (utxoset.size()) { - TxData &txd = FindRandomFrom(utxoset); + utxoData.emplace(outpoint, std::make_tuple(tx,undo,old_coin)); + } else if (utxoset.size()) { + //1/20 times undo a previous transaction + auto utxod = FindRandomFrom(utxoset); - CTransaction &tx = std::get<0>(txd); - CTxUndo &undo = std::get<1>(txd); - CCoins &origcoins = std::get<2>(txd); - - uint256 undohash = tx.GetHash(); + CTransaction &tx = std::get<0>(utxod->second); + CTxUndo &undo = std::get<1>(utxod->second); + Coin &orig_coin = std::get<2>(utxod->second); // Update the expected result // Remove new outputs - result[undohash].Clear(); + result[utxod->first].Clear(); // If not coinbase restore prevout if (!tx.IsCoinBase()) { - result[tx.vin[0].prevout.hash] = origcoins; + result[tx.vin[0].prevout] = orig_coin; } // Disconnect the tx from the current UTXO // See code in DisconnectBlock // remove outputs - { - CCoinsModifier outs = stack.back()->ModifyCoins(undohash); - outs->Clear(); - } + stack.back()->SpendCoin(utxod->first); // restore inputs if (!tx.IsCoinBase()) { const COutPoint &out = tx.vin[0].prevout; - const CTxInUndo &undoin = undo.vprevout[0]; - ApplyTxInUndo(undoin, *(stack.back()), out); + Coin coin = undo.vprevout[0]; + ApplyTxInUndo(std::move(coin), *(stack.back()), out); } // Store as a candidate for reconnection - disconnectedids.insert(undohash); + disconnected_coins.insert(utxod->first); // Update the utxoset - utxoset.erase(undohash); + utxoset.erase(utxod->first); if (!tx.IsCoinBase()) - utxoset.insert(tx.vin[0].prevout.hash); + utxoset.insert(tx.vin[0].prevout); } // Once every 1000 iterations and at the end, verify the full cache. if (insecure_rand() % 1000 == 1 || i == NUM_SIMULATION_ITERATIONS - 1) { - for (std::map<uint256, CCoins>::iterator it = result.begin(); it != result.end(); it++) { - const CCoins* coins = stack.back()->AccessCoins(it->first); - if (coins) { - BOOST_CHECK(*coins == it->second); - } else { - BOOST_CHECK(it->second.IsPruned()); - } + for (auto it = result.begin(); it != result.end(); it++) { + bool have = stack.back()->HaveCoin(it->first); + const Coin& coin = stack.back()->AccessCoin(it->first); + BOOST_CHECK(have == !coin.IsSpent()); + BOOST_CHECK(coin == it->second); } } + // One every 10 iterations, remove a random entry from the cache + if (utxoset.size() > 1 && insecure_rand() % 30 == 0) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(utxoset)->first); + } + if (disconnected_coins.size() > 1 && insecure_rand() % 30 == 0) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(disconnected_coins)->first); + } + if (duplicate_coins.size() > 1 && insecure_rand() % 30 == 0) { + stack[insecure_rand() % stack.size()]->Uncache(FindRandomFrom(duplicate_coins)->first); + } + if (insecure_rand() % 100 == 0) { // Every 100 iterations, flush an intermediate cache if (stack.size() > 1 && insecure_rand() % 2 == 0) { @@ -433,53 +469,36 @@ BOOST_AUTO_TEST_CASE(updatecoins_simulation_test) BOOST_AUTO_TEST_CASE(ccoins_serialization) { // Good example - CDataStream ss1(ParseHex("0104835800816115944e077fe7c803cfa57f29b36bf87c1d358bb85e"), SER_DISK, CLIENT_VERSION); - CCoins cc1; + CDataStream ss1(ParseHex("97f23c835800816115944e077fe7c803cfa57f29b36bf87c1d35"), SER_DISK, CLIENT_VERSION); + Coin cc1; ss1 >> cc1; - BOOST_CHECK_EQUAL(cc1.nVersion, 1); BOOST_CHECK_EQUAL(cc1.fCoinBase, false); BOOST_CHECK_EQUAL(cc1.nHeight, 203998); - BOOST_CHECK_EQUAL(cc1.vout.size(), 2); - BOOST_CHECK_EQUAL(cc1.IsAvailable(0), false); - BOOST_CHECK_EQUAL(cc1.IsAvailable(1), true); - BOOST_CHECK_EQUAL(cc1.vout[1].nValue, 60000000000ULL); - BOOST_CHECK_EQUAL(HexStr(cc1.vout[1].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); + BOOST_CHECK_EQUAL(cc1.out.nValue, 60000000000ULL); + BOOST_CHECK_EQUAL(HexStr(cc1.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("816115944e077fe7c803cfa57f29b36bf87c1d35")))))); // Good example - CDataStream ss2(ParseHex("0109044086ef97d5790061b01caab50f1b8e9c50a5057eb43c2d9563a4eebbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa486af3b"), SER_DISK, CLIENT_VERSION); - CCoins cc2; + CDataStream ss2(ParseHex("8ddf77bbd123008c988f1a4a4de2161e0f50aac7f17e7f9555caa4"), SER_DISK, CLIENT_VERSION); + Coin cc2; ss2 >> cc2; - BOOST_CHECK_EQUAL(cc2.nVersion, 1); BOOST_CHECK_EQUAL(cc2.fCoinBase, true); BOOST_CHECK_EQUAL(cc2.nHeight, 120891); - BOOST_CHECK_EQUAL(cc2.vout.size(), 17); - for (int i = 0; i < 17; i++) { - BOOST_CHECK_EQUAL(cc2.IsAvailable(i), i == 4 || i == 16); - } - BOOST_CHECK_EQUAL(cc2.vout[4].nValue, 234925952); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[4].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("61b01caab50f1b8e9c50a5057eb43c2d9563a4ee")))))); - BOOST_CHECK_EQUAL(cc2.vout[16].nValue, 110397); - BOOST_CHECK_EQUAL(HexStr(cc2.vout[16].scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); + BOOST_CHECK_EQUAL(cc2.out.nValue, 110397); + BOOST_CHECK_EQUAL(HexStr(cc2.out.scriptPubKey), HexStr(GetScriptForDestination(CKeyID(uint160(ParseHex("8c988f1a4a4de2161e0f50aac7f17e7f9555caa4")))))); // Smallest possible example - CDataStream ssx(SER_DISK, CLIENT_VERSION); - BOOST_CHECK_EQUAL(HexStr(ssx.begin(), ssx.end()), ""); - - CDataStream ss3(ParseHex("0002000600"), SER_DISK, CLIENT_VERSION); - CCoins cc3; + CDataStream ss3(ParseHex("000006"), SER_DISK, CLIENT_VERSION); + Coin cc3; ss3 >> cc3; - BOOST_CHECK_EQUAL(cc3.nVersion, 0); BOOST_CHECK_EQUAL(cc3.fCoinBase, false); BOOST_CHECK_EQUAL(cc3.nHeight, 0); - BOOST_CHECK_EQUAL(cc3.vout.size(), 1); - BOOST_CHECK_EQUAL(cc3.IsAvailable(0), true); - BOOST_CHECK_EQUAL(cc3.vout[0].nValue, 0); - BOOST_CHECK_EQUAL(cc3.vout[0].scriptPubKey.size(), 0); + BOOST_CHECK_EQUAL(cc3.out.nValue, 0); + BOOST_CHECK_EQUAL(cc3.out.scriptPubKey.size(), 0); // scriptPubKey that ends beyond the end of the stream - CDataStream ss4(ParseHex("0002000800"), SER_DISK, CLIENT_VERSION); + CDataStream ss4(ParseHex("000007"), SER_DISK, CLIENT_VERSION); try { - CCoins cc4; + Coin cc4; ss4 >> cc4; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { @@ -490,16 +509,16 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) uint64_t x = 3000000000ULL; tmp << VARINT(x); BOOST_CHECK_EQUAL(HexStr(tmp.begin(), tmp.end()), "8a95c0bb00"); - CDataStream ss5(ParseHex("0002008a95c0bb0000"), SER_DISK, CLIENT_VERSION); + CDataStream ss5(ParseHex("00008a95c0bb00"), SER_DISK, CLIENT_VERSION); try { - CCoins cc5; + Coin cc5; ss5 >> cc5; BOOST_CHECK_MESSAGE(false, "We should have thrown"); } catch (const std::ios_base::failure& e) { } } -const static uint256 TXID; +const static COutPoint OUTPOINT; const static CAmount PRUNED = -1; const static CAmount ABSENT = -2; const static CAmount FAIL = -3; @@ -514,15 +533,15 @@ const static auto FLAGS = {char(0), FRESH, DIRTY, char(DIRTY | FRESH)}; const static auto CLEAN_FLAGS = {char(0), FRESH}; const static auto ABSENT_FLAGS = {NO_ENTRY}; -void SetCoinsValue(CAmount value, CCoins& coins) +void SetCoinsValue(CAmount value, Coin& coin) { assert(value != ABSENT); - coins.Clear(); - assert(coins.IsPruned()); + coin.Clear(); + assert(coin.IsSpent()); if (value != PRUNED) { - coins.vout.emplace_back(); - coins.vout.back().nValue = value; - assert(!coins.IsPruned()); + coin.out.nValue = value; + coin.nHeight = 1; + assert(!coin.IsSpent()); } } @@ -535,25 +554,23 @@ size_t InsertCoinsMapEntry(CCoinsMap& map, CAmount value, char flags) assert(flags != NO_ENTRY); CCoinsCacheEntry entry; entry.flags = flags; - SetCoinsValue(value, entry.coins); - auto inserted = map.emplace(TXID, std::move(entry)); + SetCoinsValue(value, entry.coin); + auto inserted = map.emplace(OUTPOINT, std::move(entry)); assert(inserted.second); - return inserted.first->second.coins.DynamicMemoryUsage(); + return inserted.first->second.coin.DynamicMemoryUsage(); } void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) { - auto it = map.find(TXID); + auto it = map.find(OUTPOINT); if (it == map.end()) { value = ABSENT; flags = NO_ENTRY; } else { - if (it->second.coins.IsPruned()) { - assert(it->second.coins.vout.size() == 0); + if (it->second.coin.IsSpent()) { value = PRUNED; } else { - assert(it->second.coins.vout.size() == 1); - value = it->second.coins.vout[0].nValue; + value = it->second.coin.out.nValue; } flags = it->second.flags; assert(flags != NO_ENTRY); @@ -581,10 +598,10 @@ public: CCoinsViewCacheTest cache{&base}; }; -void CheckAccessCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckAccessCoin(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - test.cache.AccessCoins(TXID); + test.cache.AccessCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -603,39 +620,39 @@ BOOST_AUTO_TEST_CASE(ccoins_access) * Base Cache Result Cache Result * Value Value Value Flags Flags */ - CheckAccessCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoins(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, ABSENT, PRUNED, NO_ENTRY , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, 0 , 0 ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); } -void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags) +void CheckSpendCoins(CAmount base_value, CAmount cache_value, CAmount expected_value, char cache_flags, char expected_flags) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); - SetCoinsValue(modify_value, *test.cache.ModifyCoins(TXID)); + test.cache.SpendCoin(OUTPOINT); test.cache.SelfTest(); CAmount result_value; @@ -645,79 +662,55 @@ void CheckModifyCoins(CAmount base_value, CAmount cache_value, CAmount modify_va BOOST_CHECK_EQUAL(result_flags, expected_flags); }; -BOOST_AUTO_TEST_CASE(ccoins_modify) +BOOST_AUTO_TEST_CASE(ccoins_spend) { - /* Check ModifyCoin behavior, requesting a coin from a cache view layered on - * top of a base view, writing a modification to the coin, and then checking + /* Check SpendCoin behavior, requesting a coin from a cache view layered on + * top of a base view, spending, and then checking * the resulting entry in the cache after the modification. * - * Base Cache Write Result Cache Result - * Value Value Value Value Flags Flags + * Base Cache Result Cache Result + * Value Value Value Flags Flags */ - CheckModifyCoins(ABSENT, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(ABSENT, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(ABSENT, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckModifyCoins(PRUNED, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(PRUNED, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, 0 , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY , DIRTY ); - CheckModifyCoins(VALUE1, VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH); + CheckSpendCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(PRUNED, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(PRUNED, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, ABSENT, PRUNED, NO_ENTRY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); } -void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) +void CheckAddCoinBase(CAmount base_value, CAmount cache_value, CAmount modify_value, CAmount expected_value, char cache_flags, char expected_flags, bool coinbase) { SingleEntryCacheTest test(base_value, cache_value, cache_flags); CAmount result_value; char result_flags; try { - SetCoinsValue(modify_value, *test.cache.ModifyNewCoins(TXID, coinbase)); + CTxOut output; + output.nValue = modify_value; + test.cache.AddCoin(OUTPOINT, Coin(std::move(output), 1, coinbase), coinbase); + test.cache.SelfTest(); GetCoinsMapEntry(test.cache.map(), result_value, result_flags); } catch (std::logic_error& e) { result_value = FAIL; @@ -728,64 +721,46 @@ void CheckModifyNewCoinsBase(CAmount base_value, CAmount cache_value, CAmount mo BOOST_CHECK_EQUAL(result_flags, expected_flags); } -// Simple wrapper for CheckModifyNewCoinsBase function above that loops through +// Simple wrapper for CheckAddCoinBase function above that loops through // different possible base_values, making sure each one gives the same results. -// This wrapper lets the modify_new test below be shorter and less repetitive, -// while still verifying that the CoinsViewCache::ModifyNewCoins implementation +// This wrapper lets the coins_add test below be shorter and less repetitive, +// while still verifying that the CoinsViewCache::AddCoin implementation // ignores base values. template <typename... Args> -void CheckModifyNewCoins(Args&&... args) +void CheckAddCoin(Args&&... args) { for (CAmount base_value : {ABSENT, PRUNED, VALUE1}) - CheckModifyNewCoinsBase(base_value, std::forward<Args>(args)...); + CheckAddCoinBase(base_value, std::forward<Args>(args)...); } -BOOST_AUTO_TEST_CASE(ccoins_modify_new) +BOOST_AUTO_TEST_CASE(ccoins_add) { - /* Check ModifyNewCoin behavior, requesting a new coin from a cache view, + /* Check AddCoin behavior, requesting a new coin from a cache view, * writing a modification to the coin, and then checking the resulting * entry in the cache after the modification. Verify behavior with the - * with the ModifyNewCoin coinbase argument set to false, and to true. + * with the AddCoin potential_overwrite argument set to false, and to true. * - * Cache Write Result Cache Result Coinbase - * Value Value Value Flags Flags + * Cache Write Result Cache Result potential_overwrite + * Value Value Value Flags Flags */ - CheckModifyNewCoins(ABSENT, PRUNED, ABSENT, NO_ENTRY , NO_ENTRY , false); - CheckModifyNewCoins(ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); - CheckModifyNewCoins(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, 0 , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); - CheckModifyNewCoins(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, FRESH , NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, PRUNED, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, PRUNED, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckModifyNewCoins(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); - CheckModifyNewCoins(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); + CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); + CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY , NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(VALUE2, VALUE3, FAIL , DIRTY|FRESH, NO_ENTRY , false); + CheckAddCoin(VALUE2, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); } void CheckWriteCoins(CAmount parent_value, CAmount child_value, CAmount expected_value, char parent_flags, char child_flags, char expected_flags) diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 4d17417179..72e562808a 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -3,12 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "crypto/aes.h" +#include "crypto/chacha20.h" #include "crypto/ripemd160.h" #include "crypto/sha1.h" #include "crypto/sha256.h" #include "crypto/sha512.h" #include "crypto/hmac_sha256.h" #include "crypto/hmac_sha512.h" +#include "random.h" #include "utilstrencodings.h" #include "test/test_bitcoin.h" #include "test/test_random.h" @@ -187,6 +189,19 @@ void TestAES256CBC(const std::string &hexkey, const std::string &hexiv, bool pad } } +void TestChaCha20(const std::string &hexkey, uint64_t nonce, uint64_t seek, const std::string& hexout) +{ + std::vector<unsigned char> key = ParseHex(hexkey); + ChaCha20 rng(key.data(), key.size()); + rng.SetIV(nonce); + rng.Seek(seek); + std::vector<unsigned char> out = ParseHex(hexout); + std::vector<unsigned char> outres; + outres.resize(out.size()); + rng.Output(outres.data(), outres.size()); + BOOST_CHECK(out == outres); +} + std::string LongTestString(void) { std::string ret; for (int i=0; i<200000; i++) { @@ -439,4 +454,57 @@ BOOST_AUTO_TEST_CASE(aes_cbc_testvectors) { "b2eb05e2c39be9fcda6c19078c6a9d1b3f461796d6b0d6b2e0c2a72b4d80e644"); } + +BOOST_AUTO_TEST_CASE(chacha20_testvector) +{ + // Test vector from RFC 7539 + TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x4a000000UL, 1, + "224f51f3401bd9e12fde276fb8631ded8c131f823d2c06e27e4fcaec9ef3cf788a3b0aa372600a92b57974cded2b9334794cb" + "a40c63e34cdea212c4cf07d41b769a6749f3f630f4122cafe28ec4dc47e26d4346d70b98c73f3e9c53ac40c5945398b6eda1a" + "832c89c167eacd901d7e2bf363"); + + // Test vectors from https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0, 0, + "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b" + "8f41518a11cc387b669b2ee6586"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000001", 0, 0, + "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d79" + "2b1c43fea817e9ad275ae546963"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0x0100000000000000ULL, 0, + "de9cba7bf3d69ef5e786dc63973f653a0b49e015adbff7134fcb7df137821031e85a050278a7084527214f73efc7fa5b52770" + "62eb7a0433e445f41e3"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 1, 0, + "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32111e4caf237ee53ca8ad6426194a88545ddc4" + "97a0b466e7d6bbdb0041b2f586b"); + TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x0706050403020100ULL, 0, + "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3b" + "e59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc1" + "18be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5" + "a97a5f576fe064025d3ce042c566ab2c507b138db853e3d6959660996546cc9c4a6eafdc777c040d70eaf46f76dad3979e5c5" + "360c3317166a1c894c94a371876a94df7628fe4eaaf2ccb27d5aaae0ad7ad0f9d4b6ad3b54098746d4524d38407a6deb3ab78" + "fab78c9"); +} + +BOOST_AUTO_TEST_CASE(countbits_tests) +{ + FastRandomContext ctx; + for (int i = 0; i <= 64; ++i) { + if (i == 0) { + // Check handling of zero. + BOOST_CHECK_EQUAL(CountBits(0), 0); + } else if (i < 10) { + for (uint64_t j = 1 << (i - 1); (j >> i) == 0; ++j) { + // Exhaustively test up to 10 bits + BOOST_CHECK_EQUAL(CountBits(j), i); + } + } else { + for (int k = 0; k < 1000; k++) { + // Randomly test 1000 samples of each length above 10 bits. + uint64_t j = ((uint64_t)1) << (i - 1) | ctx.randbits(i - 1); + BOOST_CHECK_EQUAL(CountBits(j), i); + } + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index ccd7155627..8cae4e66e8 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> #include "cuckoocache.h" +#include "script/sigcache.h" #include "test/test_bitcoin.h" #include "random.h" #include <thread> @@ -36,20 +37,6 @@ void insecure_GetRandHash(uint256& t) *(ptr++) = insecure_rand.rand32(); } -/** Definition copied from /src/script/sigcache.cpp - */ -class uint256Hasher -{ -public: - template <uint8_t hash_select> - uint32_t operator()(const uint256& key) const - { - static_assert(hash_select <8, "SignatureCacheHasher only has 8 hashes available."); - uint32_t u; - std::memcpy(&u, key.begin() + 4 * hash_select, 4); - return u; - } -}; /* Test that no values not inserted into the cache are read out of it. @@ -59,7 +46,7 @@ public: BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) { insecure_rand = FastRandomContext(true); - CuckooCache::cache<uint256, uint256Hasher> cc{}; + CuckooCache::cache<uint256, SignatureCacheHasher> cc{}; size_t megabytes = 4; cc.setup_bytes(megabytes << 20); uint256 v; @@ -138,7 +125,7 @@ BOOST_AUTO_TEST_CASE(cuckoocache_hit_rate_ok) double HitRateThresh = 0.98; size_t megabytes = 4; for (double load = 0.1; load < 2; load *= 2) { - double hits = test_cache<CuckooCache::cache<uint256, uint256Hasher>>(megabytes, load); + double hits = test_cache<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes, load); BOOST_CHECK(normalize_hit_rate(hits, load) > HitRateThresh); } } @@ -206,7 +193,7 @@ void test_cache_erase(size_t megabytes) BOOST_AUTO_TEST_CASE(cuckoocache_erase_ok) { size_t megabytes = 4; - test_cache_erase<CuckooCache::cache<uint256, uint256Hasher>>(megabytes); + test_cache_erase<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes); } template <typename Cache> @@ -293,7 +280,7 @@ void test_cache_erase_parallel(size_t megabytes) BOOST_AUTO_TEST_CASE(cuckoocache_erase_parallel_ok) { size_t megabytes = 4; - test_cache_erase_parallel<CuckooCache::cache<uint256, uint256Hasher>>(megabytes); + test_cache_erase_parallel<CuckooCache::cache<uint256, SignatureCacheHasher>>(megabytes); } @@ -389,7 +376,7 @@ void test_cache_generations() } BOOST_AUTO_TEST_CASE(cuckoocache_generations) { - test_cache_generations<CuckooCache::cache<uint256, uint256Hasher>>(); + test_cache_generations<CuckooCache::cache<uint256, SignatureCacheHasher>>(); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/test/data/script_tests.json b/src/test/data/script_tests.json index 5c054ed3e8..e35a7ce569 100644 --- a/src/test/data/script_tests.json +++ b/src/test/data/script_tests.json @@ -349,7 +349,7 @@ ["2147483647", "0x04 0xFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], ["2147483648", "0x05 0x0000008000 EQUAL", "P2SH,STRICTENC", "OK"], ["549755813887", "0x05 0xFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], -["549755813888", "0x06 0xFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], +["549755813888", "0x06 0x000000008000 EQUAL", "P2SH,STRICTENC", "OK"], ["9223372036854775807", "0x08 0xFFFFFFFFFFFFFF7F EQUAL", "P2SH,STRICTENC", "OK"], ["-1", "0x01 0x81 EQUAL", "P2SH,STRICTENC", "OK", "Numbers are little-endian with the MSB being a sign bit"], ["-127", "0x01 0xFF EQUAL", "P2SH,STRICTENC", "OK"], diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index d8de765db1..bb7e473248 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -128,6 +128,23 @@ BOOST_AUTO_TEST_CASE(siphash) tx.nVersion = 1; ss << tx; BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); + + // Check consistency between CSipHasher and SipHashUint256[Extra]. + FastRandomContext ctx; + for (int i = 0; i < 16; ++i) { + uint64_t k1 = ctx.rand64(); + uint64_t k2 = ctx.rand64(); + uint256 x = GetRandHash(); + uint32_t n = ctx.rand32(); + uint8_t nb[4]; + WriteLE32(nb, n); + CSipHasher sip256(k1, k2); + sip256.Write(x.begin(), 32); + CSipHasher sip288 = sip256; + sip288.Write(nb, 4); + BOOST_CHECK_EQUAL(SipHashUint256(k1, k2, x), sip256.Finalize()); + BOOST_CHECK_EQUAL(SipHashUint256Extra(k1, k2, x, n), sip288.Finalize()); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/main_tests.cpp b/src/test/main_tests.cpp index d52104b4cc..656aec606b 100644 --- a/src/test/main_tests.cpp +++ b/src/test/main_tests.cpp @@ -39,17 +39,18 @@ static void TestBlockSubsidyHalvings(int nSubsidyHalvingInterval) BOOST_AUTO_TEST_CASE(block_subsidy_test) { - TestBlockSubsidyHalvings(Params(CBaseChainParams::MAIN).GetConsensus()); // As in main + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + TestBlockSubsidyHalvings(chainParams->GetConsensus()); // As in main TestBlockSubsidyHalvings(150); // As in regtest TestBlockSubsidyHalvings(1000); // Just another interval } BOOST_AUTO_TEST_CASE(subsidy_limit_test) { - const Consensus::Params& consensusParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); CAmount nSum = 0; for (int nHeight = 0; nHeight < 14000000; nHeight += 1000) { - CAmount nSubsidy = GetBlockSubsidy(nHeight, consensusParams); + CAmount nSubsidy = GetBlockSubsidy(nHeight, chainParams->GetConsensus()); BOOST_CHECK(nSubsidy <= 50 * COIN); nSum += nSubsidy * 1000; BOOST_CHECK(MoneyRange(nSum)); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 41f42c7b88..a40060e657 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -6,6 +6,7 @@ #include "coins.h" #include "consensus/consensus.h" #include "consensus/merkle.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "validation.h" #include "miner.h" @@ -194,7 +195,8 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) { // Note that by default, these tests run with size accounting enabled. - const CChainParams& chainparams = Params(CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const CChainParams& chainparams = *chainParams; CScript scriptPubKey = CScript() << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f") << OP_CHECKSIG; std::unique_ptr<CBlockTemplate> pblocktemplate; CMutableTransaction tx,tx2; diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index b9ed4952bb..66354699b2 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -11,6 +11,7 @@ #include "net.h" #include "netbase.h" #include "chainparams.h" +#include "util.h" class CAddrManSerializationMock : public CAddrMan { @@ -72,6 +73,18 @@ CDataStream AddrmanToStream(CAddrManSerializationMock& _addrman) BOOST_FIXTURE_TEST_SUITE(net_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(cnode_listen_port) +{ + // test default + unsigned short port = GetListenPort(); + BOOST_CHECK(port == Params().GetDefaultPort()); + // test set port + unsigned short altPort = 12345; + SoftSetArg("-port", std::to_string(altPort)); + port = GetListenPort(); + BOOST_CHECK(port == altPort); +} + BOOST_AUTO_TEST_CASE(caddrdb_read) { CAddrManUncorrupted addrmanUncorrupted; @@ -162,12 +175,12 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) bool fInboundIn = false; // Test that fFeeler is false by default. - std::unique_ptr<CNode> pnode1(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 0, 0, pszDest, fInboundIn)); + std::unique_ptr<CNode> pnode1(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 0, 0, CAddress(), pszDest, fInboundIn)); BOOST_CHECK(pnode1->fInbound == false); BOOST_CHECK(pnode1->fFeeler == false); fInboundIn = true; - std::unique_ptr<CNode> pnode2(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 1, 1, pszDest, fInboundIn)); + std::unique_ptr<CNode> pnode2(new CNode(id++, NODE_NETWORK, height, hSocket, addr, 1, 1, CAddress(), pszDest, fInboundIn)); BOOST_CHECK(pnode2->fInbound == true); BOOST_CHECK(pnode2->fFeeler == false); } diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index bc2f49ef3f..6bfd315647 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -16,7 +16,8 @@ BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { - CTxMemPool mpool; + CBlockPolicyEstimator feeEst; + CTxMemPool mpool(&feeEst); TestMemPoolEntryHelper entry; CAmount basefee(2000); CAmount deltaFee(100); @@ -49,8 +50,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) int blocknum = 0; // Loop through 200 blocks - // At a decay .998 and 4 fee transactions per block - // This makes the tx count about 1.33 per bucket, above the 1 threshold + // At a decay .9952 and 4 fee transactions per block + // This makes the tx count about 2.5 per bucket, well above the 0.1 threshold while (blocknum < 200) { for (int j = 0; j < 10; j++) { // For each fee for (int k = 0; k < 4; k++) { // add 4 fee txs @@ -74,20 +75,14 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } mpool.removeForBlock(block, ++blocknum); block.clear(); - if (blocknum == 30) { - // At this point we should need to combine 5 buckets to get enough data points - // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around - // 8*baserate. estimateFee(4) %'s are 100,100,100,100,90 = average 98% - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(2) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(3) == CFeeRate(0)); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() < 8*baseRate.GetFeePerK() + deltaFee); - BOOST_CHECK(mpool.estimateFee(4).GetFeePerK() > 8*baseRate.GetFeePerK() - deltaFee); - int answerFound; - BOOST_CHECK(mpool.estimateSmartFee(1, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(3, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(4, &answerFound) == mpool.estimateFee(4) && answerFound == 4); - BOOST_CHECK(mpool.estimateSmartFee(8, &answerFound) == mpool.estimateFee(8) && answerFound == 8); + // Check after just a few txs that combining buckets works as expected + if (blocknum == 3) { + // At this point we should need to combine 3 buckets to get enough data points + // So estimateFee(1) should fail and estimateFee(2) should return somewhere around + // 9*baserate. estimateFee(2) %'s are 100,100,90 = average 97% + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() < 9*baseRate.GetFeePerK() + deltaFee); + BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() > 9*baseRate.GetFeePerK() - deltaFee); } } @@ -99,18 +94,19 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // Second highest feerate has 100% chance of being included by 2 blocks, // so estimateFee(2) should return 9*baseRate etc... for (int i = 1; i < 10;i++) { - origFeeEst.push_back(mpool.estimateFee(i).GetFeePerK()); + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); if (i > 2) { // Fee estimates should be monotonically decreasing BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]); } int mult = 11-i; - if (i > 1) { + if (i % 2 == 0) { //At scale 2, test logic is only correct for even targets BOOST_CHECK(origFeeEst[i-1] < mult*baseRate.GetFeePerK() + deltaFee); BOOST_CHECK(origFeeEst[i-1] > mult*baseRate.GetFeePerK() - deltaFee); } - else { - BOOST_CHECK(origFeeEst[i-1] == CFeeRate(0).GetFeePerK()); - } + } + // Fill out rest of the original estimates + for (int i = 10; i <= 48; i++) { + origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK()); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change @@ -118,10 +114,10 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) while (blocknum < 250) mpool.removeForBlock(block, ++blocknum); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee); + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } @@ -139,10 +135,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); } - int answerFound; for (int i = 1; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i) == CFeeRate(0) || mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); - BOOST_CHECK(mpool.estimateSmartFee(i, &answerFound).GetFeePerK() > origFeeEst[answerFound-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } // Mine all those transactions @@ -155,16 +149,16 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[j].pop_back(); } } - mpool.removeForBlock(block, 265); + mpool.removeForBlock(block, 266); block.clear(); - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } - // Mine 200 more blocks where everything is mined every block + // Mine 400 more blocks where everything is mined every block // Estimates should be below original estimates - while (blocknum < 465) { + while (blocknum < 665) { for (int j = 0; j < 10; j++) { // For each fee multiple for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; @@ -179,9 +173,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.removeForBlock(block, ++blocknum); block.clear(); } - BOOST_CHECK(mpool.estimateFee(1) == CFeeRate(0)); - for (int i = 2; i < 10; i++) { - BOOST_CHECK(mpool.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); + for (int i = 2; i < 9; i++) { // At 9, the original estimate was already at the bottom (b/c scale = 2) + BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); } // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee @@ -190,8 +184,8 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) mpool.TrimToSize(1); BOOST_CHECK(mpool.GetMinFee(1).GetFeePerK() > feeV[5]); for (int i = 1; i < 10; i++) { - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.estimateFee(i).GetFeePerK()); - BOOST_CHECK(mpool.estimateSmartFee(i).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= feeEst.estimateRawFee(i, 0.85, FeeEstimateHorizon::MED_HALFLIFE).GetFeePerK()); + BOOST_CHECK(feeEst.estimateSmartFee(i, NULL, mpool).GetFeePerK() >= mpool.GetMinFee(1).GetFeePerK()); } } diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index 4ca6f1caf0..3b79f8000d 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -16,69 +16,59 @@ BOOST_FIXTURE_TEST_SUITE(pow_tests, BasicTestingSetup) /* Test calculation of next difficulty target with no constraints applying */ BOOST_AUTO_TEST_CASE(get_next_work) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1261130161; // Block #30240 CBlockIndex pindexLast; pindexLast.nHeight = 32255; pindexLast.nTime = 1262152739; // Block #32255 pindexLast.nBits = 0x1d00ffff; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00d86a); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00d86a); } /* Test the constraint on the upper bound for next work */ BOOST_AUTO_TEST_CASE(get_next_work_pow_limit) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1231006505; // Block #0 CBlockIndex pindexLast; pindexLast.nHeight = 2015; pindexLast.nTime = 1233061996; // Block #2015 pindexLast.nBits = 0x1d00ffff; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00ffff); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00ffff); } /* Test the constraint on the lower bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_lower_limit_actual) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1279008237; // Block #66528 CBlockIndex pindexLast; pindexLast.nHeight = 68543; pindexLast.nTime = 1279297671; // Block #68543 pindexLast.nBits = 0x1c05a3f4; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1c0168fd); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1c0168fd); } /* Test the constraint on the upper bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_upper_limit_actual) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); int64_t nLastRetargetTime = 1263163443; // NOTE: Not an actual block time CBlockIndex pindexLast; pindexLast.nHeight = 46367; pindexLast.nTime = 1269211443; // Block #46367 pindexLast.nBits = 0x1c387f6f; - BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, params), 0x1d00e1fd); + BOOST_CHECK_EQUAL(CalculateNextWorkRequired(&pindexLast, nLastRetargetTime, chainParams->GetConsensus()), 0x1d00e1fd); } BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) { - SelectParams(CBaseChainParams::MAIN); - const Consensus::Params& params = Params().GetConsensus(); - + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); std::vector<CBlockIndex> blocks(10000); for (int i = 0; i < 10000; i++) { blocks[i].pprev = i ? &blocks[i - 1] : NULL; blocks[i].nHeight = i; - blocks[i].nTime = 1269211443 + i * params.nPowTargetSpacing; + blocks[i].nTime = 1269211443 + i * chainParams->GetConsensus().nPowTargetSpacing; blocks[i].nBits = 0x207fffff; /* target 0x7fffff000... */ blocks[i].nChainWork = i ? blocks[i - 1].nChainWork + GetBlockProof(blocks[i - 1]) : arith_uint256(0); } @@ -88,7 +78,7 @@ BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) CBlockIndex *p2 = &blocks[GetRand(10000)]; CBlockIndex *p3 = &blocks[GetRand(10000)]; - int64_t tdiff = GetBlockProofEquivalentTime(*p1, *p2, *p3, params); + int64_t tdiff = GetBlockProofEquivalentTime(*p1, *p2, *p3, chainParams->GetConsensus()); BOOST_CHECK_EQUAL(tdiff, p1->GetBlockTime() - p2->GetBlockTime()); } } diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index bd8a7819a4..cfed5e347e 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -28,6 +28,7 @@ class prevector_tester { typedef typename pretype::size_type Size; bool passed = true; FastRandomContext rand_cache; + uint256 rand_seed; template <typename A, typename B> @@ -183,13 +184,12 @@ public: } ~prevector_tester() { - BOOST_CHECK_MESSAGE(passed, "insecure_rand_Rz: " - << rand_cache.Rz - << ", insecure_rand_Rw: " - << rand_cache.Rw); + BOOST_CHECK_MESSAGE(passed, "insecure_rand: " + rand_seed.ToString()); } + prevector_tester() { seed_insecure_rand(); + rand_seed = insecure_rand_seed; rand_cache = insecure_rand_ctx; } }; diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index d2c46c0daa..8596734226 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -15,5 +15,39 @@ BOOST_AUTO_TEST_CASE(osrandom_tests) BOOST_CHECK(Random_SanityCheck()); } -BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_CASE(fastrandom_tests) +{ + // Check that deterministic FastRandomContexts are deterministic + FastRandomContext ctx1(true); + FastRandomContext ctx2(true); + + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.rand64(), ctx2.rand64()); + BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); + BOOST_CHECK_EQUAL(ctx1.randbits(7), ctx2.randbits(7)); + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); + + // Check that a nondeterministic ones are not + FastRandomContext ctx3; + FastRandomContext ctx4; + BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal +} +BOOST_AUTO_TEST_CASE(fastrandom_randbits) +{ + FastRandomContext ctx1; + FastRandomContext ctx2; + for (int bits = 0; bits < 63; ++bits) { + for (int j = 0; j < 1000; ++j) { + uint64_t rangebits = ctx1.randbits(bits); + BOOST_CHECK_EQUAL(rangebits >> bits, 0); + uint64_t range = ((uint64_t)1) << bits | rangebits; + uint64_t rand = ctx2.randrange(range); + BOOST_CHECK(rand < range); + } + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_P2SH_tests.cpp index f8fd8cc30c..0789b2e80c 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_P2SH_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "consensus/tx_verify.h" #include "core_io.h" #include "key.h" #include "keystore.h" @@ -111,7 +112,8 @@ BOOST_AUTO_TEST_CASE(sign) { CScript sigSave = txTo[i].vin[0].scriptSig; txTo[i].vin[0].scriptSig = txTo[j].vin[0].scriptSig; - bool sigOK = CScriptCheck(CCoins(txFrom, 0), txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); + const CTxOut& output = txFrom.vout[txTo[i].vin[0].prevout.n]; + bool sigOK = CScriptCheck(output.scriptPubKey, output.nValue, txTo[i], 0, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC, false, &txdata)(); if (i == j) BOOST_CHECK_MESSAGE(sigOK, strprintf("VerifySignature %d %d", i, j)); else @@ -315,7 +317,7 @@ BOOST_AUTO_TEST_CASE(AreInputsStandard) txFrom.vout[6].scriptPubKey = GetScriptForDestination(CScriptID(twentySigops)); txFrom.vout[6].nValue = 6000; - coins.ModifyCoins(txFrom.GetHash())->FromTx(txFrom, 0); + AddCoins(coins, txFrom, 0); CMutableTransaction txTo; txTo.vout.resize(1); diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 5279cb243b..2f7c22084e 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "data/sighash.json.h" #include "hash.h" -#include "validation.h" // For CheckTransaction #include "script/interpreter.h" #include "script/script.h" #include "serialize.h" diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 13d8911f03..4e117448fe 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "validation.h" +#include "consensus/tx_verify.h" #include "pubkey.h" #include "key.h" #include "script/script.h" @@ -102,7 +102,7 @@ void BuildTxs(CMutableTransaction& spendingTx, CCoinsViewCache& coins, CMutableT spendingTx.vout[0].nValue = 1; spendingTx.vout[0].scriptPubKey = CScript(); - coins.ModifyCoins(creationTx.GetHash())->FromTx(creationTx, 0); + AddCoins(coins, creationTx, 0); } BOOST_AUTO_TEST_CASE(GetTxSigOpCost) diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index bda3819662..cb625bda11 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -27,7 +27,8 @@ #include <boost/thread.hpp> -FastRandomContext insecure_rand_ctx(true); +uint256 insecure_rand_seed = GetRandHash(); +FastRandomContext insecure_rand_ctx(insecure_rand_seed); extern bool fPrintToConsole; extern void noui_connect(); diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/test_bitcoin_fuzzy.cpp index c4983f6f5c..de14251601 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/test_bitcoin_fuzzy.cpp @@ -59,9 +59,8 @@ bool read_stdin(std::vector<char> &data) { return length==0; } -int main(int argc, char **argv) +int do_fuzz() { - ECCVerifyHandle globalVerifyHandle; std::vector<char> buffer; if (!read_stdin(buffer)) return 0; @@ -169,8 +168,8 @@ int main(int argc, char **argv) { try { - CCoins block; - ds >> block; + Coin coin; + ds >> coin; } catch (const std::ios_base::failure& e) {return 0;} break; } @@ -256,3 +255,23 @@ int main(int argc, char **argv) return 0; } +int main(int argc, char **argv) +{ + ECCVerifyHandle globalVerifyHandle; +#ifdef __AFL_INIT + // Enable AFL deferred forkserver mode. Requires compilation using + // afl-clang-fast++. See fuzzing.md for details. + __AFL_INIT(); +#endif + +#ifdef __AFL_LOOP + // Enable AFL persistent mode. Requires compilation using afl-clang-fast++. + // See fuzzing.md for details. + while (__AFL_LOOP(1000)) { + do_fuzz(); + } + return 0; +#else + return do_fuzz(); +#endif +} diff --git a/src/test/test_random.h b/src/test/test_random.h index 4a1637ac72..318c44df4d 100644 --- a/src/test/test_random.h +++ b/src/test/test_random.h @@ -8,11 +8,17 @@ #include "random.h" +extern uint256 insecure_rand_seed; extern FastRandomContext insecure_rand_ctx; static inline void seed_insecure_rand(bool fDeterministic = false) { - insecure_rand_ctx = FastRandomContext(fDeterministic); + if (fDeterministic) { + insecure_rand_seed = uint256(); + } else { + insecure_rand_seed = GetRandHash(); + } + insecure_rand_ctx = FastRandomContext(insecure_rand_seed); } static inline uint32_t insecure_rand(void) diff --git a/src/test/torcontrol_tests.cpp b/src/test/torcontrol_tests.cpp new file mode 100644 index 0000000000..b7affaacde --- /dev/null +++ b/src/test/torcontrol_tests.cpp @@ -0,0 +1,199 @@ +// Copyright (c) 2017 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include "test/test_bitcoin.h" +#include "torcontrol.cpp" + +#include <boost/test/unit_test.hpp> + + +BOOST_FIXTURE_TEST_SUITE(torcontrol_tests, BasicTestingSetup) + +void CheckSplitTorReplyLine(std::string input, std::string command, std::string args) +{ + BOOST_TEST_MESSAGE(std::string("CheckSplitTorReplyLine(") + input + ")"); + auto ret = SplitTorReplyLine(input); + BOOST_CHECK_EQUAL(ret.first, command); + BOOST_CHECK_EQUAL(ret.second, args); +} + +BOOST_AUTO_TEST_CASE(util_SplitTorReplyLine) +{ + // Data we should receive during normal usage + CheckSplitTorReplyLine( + "PROTOCOLINFO PIVERSION", + "PROTOCOLINFO", "PIVERSION"); + CheckSplitTorReplyLine( + "AUTH METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\"", + "AUTH", "METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\""); + CheckSplitTorReplyLine( + "AUTH METHODS=NULL", + "AUTH", "METHODS=NULL"); + CheckSplitTorReplyLine( + "AUTH METHODS=HASHEDPASSWORD", + "AUTH", "METHODS=HASHEDPASSWORD"); + CheckSplitTorReplyLine( + "VERSION Tor=\"0.2.9.8 (git-a0df013ea241b026)\"", + "VERSION", "Tor=\"0.2.9.8 (git-a0df013ea241b026)\""); + CheckSplitTorReplyLine( + "AUTHCHALLENGE SERVERHASH=aaaa SERVERNONCE=bbbb", + "AUTHCHALLENGE", "SERVERHASH=aaaa SERVERNONCE=bbbb"); + + // Other valid inputs + CheckSplitTorReplyLine("COMMAND", "COMMAND", ""); + CheckSplitTorReplyLine("COMMAND SOME ARGS", "COMMAND", "SOME ARGS"); + + // These inputs are valid because PROTOCOLINFO accepts an OtherLine that is + // just an OptArguments, which enables multiple spaces to be present + // between the command and arguments. + CheckSplitTorReplyLine("COMMAND ARGS", "COMMAND", " ARGS"); + CheckSplitTorReplyLine("COMMAND EVEN+more ARGS", "COMMAND", " EVEN+more ARGS"); +} + +void CheckParseTorReplyMapping(std::string input, std::map<std::string,std::string> expected) +{ + BOOST_TEST_MESSAGE(std::string("CheckParseTorReplyMapping(") + input + ")"); + auto ret = ParseTorReplyMapping(input); + BOOST_CHECK_EQUAL(ret.size(), expected.size()); + auto r_it = ret.begin(); + auto e_it = expected.begin(); + while (r_it != ret.end() && e_it != expected.end()) { + BOOST_CHECK_EQUAL(r_it->first, e_it->first); + BOOST_CHECK_EQUAL(r_it->second, e_it->second); + r_it++; + e_it++; + } +} + +BOOST_AUTO_TEST_CASE(util_ParseTorReplyMapping) +{ + // Data we should receive during normal usage + CheckParseTorReplyMapping( + "METHODS=COOKIE,SAFECOOKIE COOKIEFILE=\"/home/x/.tor/control_auth_cookie\"", { + {"METHODS", "COOKIE,SAFECOOKIE"}, + {"COOKIEFILE", "/home/x/.tor/control_auth_cookie"}, + }); + CheckParseTorReplyMapping( + "METHODS=NULL", { + {"METHODS", "NULL"}, + }); + CheckParseTorReplyMapping( + "METHODS=HASHEDPASSWORD", { + {"METHODS", "HASHEDPASSWORD"}, + }); + CheckParseTorReplyMapping( + "Tor=\"0.2.9.8 (git-a0df013ea241b026)\"", { + {"Tor", "0.2.9.8 (git-a0df013ea241b026)"}, + }); + CheckParseTorReplyMapping( + "SERVERHASH=aaaa SERVERNONCE=bbbb", { + {"SERVERHASH", "aaaa"}, + {"SERVERNONCE", "bbbb"}, + }); + CheckParseTorReplyMapping( + "ServiceID=exampleonion1234", { + {"ServiceID", "exampleonion1234"}, + }); + CheckParseTorReplyMapping( + "PrivateKey=RSA1024:BLOB", { + {"PrivateKey", "RSA1024:BLOB"}, + }); + CheckParseTorReplyMapping( + "ClientAuth=bob:BLOB", { + {"ClientAuth", "bob:BLOB"}, + }); + + // Other valid inputs + CheckParseTorReplyMapping( + "Foo=Bar=Baz Spam=Eggs", { + {"Foo", "Bar=Baz"}, + {"Spam", "Eggs"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar=Baz\"", { + {"Foo", "Bar=Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar Baz\"", { + {"Foo", "Bar Baz"}, + }); + + // Escapes + CheckParseTorReplyMapping( + "Foo=\"Bar\\ Baz\"", { + {"Foo", "Bar Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\Baz\"", { + {"Foo", "BarBaz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\@Baz\"", { + {"Foo", "Bar@Baz"}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\\"Baz\" Spam=\"\\\"Eggs\\\"\"", { + {"Foo", "Bar\"Baz"}, + {"Spam", "\"Eggs\""}, + }); + CheckParseTorReplyMapping( + "Foo=\"Bar\\\\Baz\"", { + {"Foo", "Bar\\Baz"}, + }); + + // C escapes + CheckParseTorReplyMapping( + "Foo=\"Bar\\nBaz\\t\" Spam=\"\\rEggs\" Octals=\"\\1a\\11\\17\\18\\81\\377\\378\\400\\2222\" Final=Check", { + {"Foo", "Bar\nBaz\t"}, + {"Spam", "\rEggs"}, + {"Octals", "\1a\11\17\1" "881\377\37" "8\40" "0\222" "2"}, + {"Final", "Check"}, + }); + CheckParseTorReplyMapping( + "Valid=Mapping Escaped=\"Escape\\\\\"", { + {"Valid", "Mapping"}, + {"Escaped", "Escape\\"}, + }); + CheckParseTorReplyMapping( + "Valid=Mapping Bare=\"Escape\\\"", {}); + CheckParseTorReplyMapping( + "OneOctal=\"OneEnd\\1\" TwoOctal=\"TwoEnd\\11\"", { + {"OneOctal", "OneEnd\1"}, + {"TwoOctal", "TwoEnd\11"}, + }); + + // Special handling for null case + // (needed because string comparison reads the null as end-of-string) + BOOST_TEST_MESSAGE(std::string("CheckParseTorReplyMapping(Null=\"\\0\")")); + auto ret = ParseTorReplyMapping("Null=\"\\0\""); + BOOST_CHECK_EQUAL(ret.size(), 1); + auto r_it = ret.begin(); + BOOST_CHECK_EQUAL(r_it->first, "Null"); + BOOST_CHECK_EQUAL(r_it->second.size(), 1); + BOOST_CHECK_EQUAL(r_it->second[0], '\0'); + + // A more complex valid grammar. PROTOCOLINFO accepts a VersionLine that + // takes a key=value pair followed by an OptArguments, making this valid. + // Because an OptArguments contains no semantic data, there is no point in + // parsing it. + CheckParseTorReplyMapping( + "SOME=args,here MORE optional=arguments here", { + {"SOME", "args,here"}, + }); + + // Inputs that are effectively invalid under the target grammar. + // PROTOCOLINFO accepts an OtherLine that is just an OptArguments, which + // would make these inputs valid. However, + // - This parser is never used in that situation, because the + // SplitTorReplyLine parser enables OtherLine to be skipped. + // - Even if these were valid, an OptArguments contains no semantic data, + // so there is no point in parsing it. + CheckParseTorReplyMapping("ARGS", {}); + CheckParseTorReplyMapping("MORE ARGS", {}); + CheckParseTorReplyMapping("MORE ARGS", {}); + CheckParseTorReplyMapping("EVEN more=ARGS", {}); + CheckParseTorReplyMapping("EVEN+more ARGS", {}); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 3b5da4980b..5c7516fbf1 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -8,11 +8,12 @@ #include "clientversion.h" #include "checkqueue.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "core_io.h" #include "key.h" #include "keystore.h" -#include "validation.h" // For CheckTransaction +#include "validation.h" #include "policy/policy.h" #include "script/script.h" #include "script/sign.h" @@ -306,14 +307,14 @@ SetupDummyInputs(CBasicKeyStore& keystoreRet, CCoinsViewCache& coinsRet) dummyTransactions[0].vout[0].scriptPubKey << ToByteVector(key[0].GetPubKey()) << OP_CHECKSIG; dummyTransactions[0].vout[1].nValue = 50*CENT; dummyTransactions[0].vout[1].scriptPubKey << ToByteVector(key[1].GetPubKey()) << OP_CHECKSIG; - coinsRet.ModifyCoins(dummyTransactions[0].GetHash())->FromTx(dummyTransactions[0], 0); + AddCoins(coinsRet, dummyTransactions[0], 0); dummyTransactions[1].vout.resize(2); dummyTransactions[1].vout[0].nValue = 21*CENT; dummyTransactions[1].vout[0].scriptPubKey = GetScriptForDestination(key[2].GetPubKey().GetID()); dummyTransactions[1].vout[1].nValue = 22*CENT; dummyTransactions[1].vout[1].scriptPubKey = GetScriptForDestination(key[3].GetPubKey().GetID()); - coinsRet.ModifyCoins(dummyTransactions[1].GetHash())->FromTx(dummyTransactions[1], 0); + AddCoins(coinsRet, dummyTransactions[1], 0); return dummyTransactions; } @@ -469,19 +470,20 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) { for (int i=0; i<20; i++) threadGroup.create_thread(boost::bind(&CCheckQueue<CScriptCheck>::Thread, boost::ref(scriptcheckqueue))); - CCoins coins; - coins.nVersion = 1; - coins.fCoinBase = false; + std::vector<Coin> coins; for(uint32_t i = 0; i < mtx.vin.size(); i++) { - CTxOut txout; - txout.nValue = 1000; - txout.scriptPubKey = scriptPubKey; - coins.vout.push_back(txout); + Coin coin; + coin.nHeight = 1; + coin.fCoinBase = false; + coin.out.nValue = 1000; + coin.out.scriptPubKey = scriptPubKey; + coins.emplace_back(std::move(coin)); } for(uint32_t i = 0; i < mtx.vin.size(); i++) { std::vector<CScriptCheck> vChecks; - CScriptCheck check(coins, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); + const CTxOut& output = coins[tx.vin[i].prevout.n].out; + CScriptCheck check(output.scriptPubKey, output.nValue, tx, i, SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, false, &txdata); vChecks.push_back(CScriptCheck()); check.swap(vChecks.back()); control.Add(vChecks); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 5da4907f06..10330c0c23 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -17,8 +17,6 @@ #include <boost/test/unit_test.hpp> -extern std::map<std::string, std::string> mapArgs; - BOOST_FIXTURE_TEST_SUITE(util_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(util_criticalsection) @@ -100,52 +98,67 @@ BOOST_AUTO_TEST_CASE(util_DateTimeStrFormat) BOOST_CHECK_EQUAL(DateTimeStrFormat("%a, %d %b %Y %H:%M:%S +0000", 1317425777), "Fri, 30 Sep 2011 23:36:17 +0000"); } +class TestArgsManager : public ArgsManager +{ +public: + std::map<std::string, std::string>& GetMapArgs() + { + return mapArgs; + }; + const std::map<std::string, std::vector<std::string> >& GetMapMultiArgs() + { + return mapMultiArgs; + }; +}; + BOOST_AUTO_TEST_CASE(util_ParseParameters) { + TestArgsManager testArgs; const char *argv_test[] = {"-ignored", "-a", "-b", "-ccc=argument", "-ccc=multiple", "f", "-d=e"}; - ParseParameters(0, (char**)argv_test); - BOOST_CHECK(mapArgs.empty() && mapMultiArgs.empty()); + testArgs.ParseParameters(0, (char**)argv_test); + BOOST_CHECK(testArgs.GetMapArgs().empty() && testArgs.GetMapMultiArgs().empty()); - ParseParameters(1, (char**)argv_test); - BOOST_CHECK(mapArgs.empty() && mapMultiArgs.empty()); + testArgs.ParseParameters(1, (char**)argv_test); + BOOST_CHECK(testArgs.GetMapArgs().empty() && testArgs.GetMapMultiArgs().empty()); - ParseParameters(5, (char**)argv_test); + testArgs.ParseParameters(5, (char**)argv_test); // expectation: -ignored is ignored (program name argument), // -a, -b and -ccc end up in map, -d ignored because it is after // a non-option argument (non-GNU option parsing) - BOOST_CHECK(mapArgs.size() == 3 && mapMultiArgs.size() == 3); - BOOST_CHECK(IsArgSet("-a") && IsArgSet("-b") && IsArgSet("-ccc") - && !IsArgSet("f") && !IsArgSet("-d")); - BOOST_CHECK(mapMultiArgs.count("-a") && mapMultiArgs.count("-b") && mapMultiArgs.count("-ccc") - && !mapMultiArgs.count("f") && !mapMultiArgs.count("-d")); - - BOOST_CHECK(mapArgs["-a"] == "" && mapArgs["-ccc"] == "multiple"); - BOOST_CHECK(mapMultiArgs.at("-ccc").size() == 2); + BOOST_CHECK(testArgs.GetMapArgs().size() == 3 && testArgs.GetMapMultiArgs().size() == 3); + BOOST_CHECK(testArgs.IsArgSet("-a") && testArgs.IsArgSet("-b") && testArgs.IsArgSet("-ccc") + && !testArgs.IsArgSet("f") && !testArgs.IsArgSet("-d")); + BOOST_CHECK(testArgs.GetMapMultiArgs().count("-a") && testArgs.GetMapMultiArgs().count("-b") && testArgs.GetMapMultiArgs().count("-ccc") + && !testArgs.GetMapMultiArgs().count("f") && !testArgs.GetMapMultiArgs().count("-d")); + + BOOST_CHECK(testArgs.GetMapArgs()["-a"] == "" && testArgs.GetMapArgs()["-ccc"] == "multiple"); + BOOST_CHECK(testArgs.GetArgs("-ccc").size() == 2); } BOOST_AUTO_TEST_CASE(util_GetArg) { - mapArgs.clear(); - mapArgs["strtest1"] = "string..."; + TestArgsManager testArgs; + testArgs.GetMapArgs().clear(); + testArgs.GetMapArgs()["strtest1"] = "string..."; // strtest2 undefined on purpose - mapArgs["inttest1"] = "12345"; - mapArgs["inttest2"] = "81985529216486895"; + testArgs.GetMapArgs()["inttest1"] = "12345"; + testArgs.GetMapArgs()["inttest2"] = "81985529216486895"; // inttest3 undefined on purpose - mapArgs["booltest1"] = ""; + testArgs.GetMapArgs()["booltest1"] = ""; // booltest2 undefined on purpose - mapArgs["booltest3"] = "0"; - mapArgs["booltest4"] = "1"; - - BOOST_CHECK_EQUAL(GetArg("strtest1", "default"), "string..."); - BOOST_CHECK_EQUAL(GetArg("strtest2", "default"), "default"); - BOOST_CHECK_EQUAL(GetArg("inttest1", -1), 12345); - BOOST_CHECK_EQUAL(GetArg("inttest2", -1), 81985529216486895LL); - BOOST_CHECK_EQUAL(GetArg("inttest3", -1), -1); - BOOST_CHECK_EQUAL(GetBoolArg("booltest1", false), true); - BOOST_CHECK_EQUAL(GetBoolArg("booltest2", false), false); - BOOST_CHECK_EQUAL(GetBoolArg("booltest3", false), false); - BOOST_CHECK_EQUAL(GetBoolArg("booltest4", false), true); + testArgs.GetMapArgs()["booltest3"] = "0"; + testArgs.GetMapArgs()["booltest4"] = "1"; + + BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); + BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest1", -1), 12345); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest2", -1), 81985529216486895LL); + BOOST_CHECK_EQUAL(testArgs.GetArg("inttest3", -1), -1); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest1", false), true); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest2", false), false); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest3", false), false); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest4", false), true); } BOOST_AUTO_TEST_CASE(util_FormatMoney) diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index e2b5573abd..79405ec4d1 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -209,7 +209,8 @@ BOOST_AUTO_TEST_CASE(versionbits_test) } // Sanity checks of version bit deployments - const Consensus::Params &mainnetParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const Consensus::Params &mainnetParams = chainParams->GetConsensus(); for (int i=0; i<(int) Consensus::MAX_VERSION_BITS_DEPLOYMENTS; i++) { uint32_t bitmask = VersionBitsMask(mainnetParams, (Consensus::DeploymentPos)i); // Make sure that no deployment tries to set an invalid bit. @@ -235,7 +236,8 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) { // Check that ComputeBlockVersion will set the appropriate bit correctly // on mainnet. - const Consensus::Params &mainnetParams = Params(CBaseChainParams::MAIN).GetConsensus(); + const auto chainParams = CreateChainParams(CBaseChainParams::MAIN); + const Consensus::Params &mainnetParams = chainParams->GetConsensus(); // Use the TESTDUMMY deployment for testing purposes. int64_t bit = mainnetParams.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].bit; diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index c1bd95b00f..8a37139f1d 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -1,4 +1,5 @@ // Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2017 The Zcash developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,7 +15,6 @@ #include <set> #include <stdlib.h> -#include <boost/function.hpp> #include <boost/bind.hpp> #include <boost/signals2/signal.hpp> #include <boost/foreach.hpp> @@ -73,8 +73,8 @@ public: class TorControlConnection { public: - typedef boost::function<void(TorControlConnection&)> ConnectionCB; - typedef boost::function<void(TorControlConnection &,const TorControlReply &)> ReplyHandlerCB; + typedef std::function<void(TorControlConnection&)> ConnectionCB; + typedef std::function<void(TorControlConnection &,const TorControlReply &)> ReplyHandlerCB; /** Create a new TorControlConnection. */ @@ -105,9 +105,9 @@ public: boost::signals2::signal<void(TorControlConnection &,const TorControlReply &)> async_handler; private: /** Callback when ready for use */ - boost::function<void(TorControlConnection&)> connected; + std::function<void(TorControlConnection&)> connected; /** Callback when connection lost */ - boost::function<void(TorControlConnection&)> disconnected; + std::function<void(TorControlConnection&)> disconnected; /** Libevent event base */ struct event_base *base; /** Connection to control socket */ @@ -250,6 +250,8 @@ bool TorControlConnection::Command(const std::string &cmd, const ReplyHandlerCB& /* Split reply line in the form 'AUTH METHODS=...' into a type * 'AUTH' and arguments 'METHODS=...'. + * Grammar is implicitly defined in https://spec.torproject.org/control-spec by + * the server reply formats for PROTOCOLINFO (S3.21) and AUTHCHALLENGE (S3.24). */ static std::pair<std::string,std::string> SplitTorReplyLine(const std::string &s) { @@ -265,6 +267,10 @@ static std::pair<std::string,std::string> SplitTorReplyLine(const std::string &s } /** Parse reply arguments in the form 'METHODS=COOKIE,SAFECOOKIE COOKIEFILE=".../control_auth_cookie"'. + * Returns a map of keys to values, or an empty map if there was an error. + * Grammar is implicitly defined in https://spec.torproject.org/control-spec by + * the server reply formats for PROTOCOLINFO (S3.21), AUTHCHALLENGE (S3.24), + * and ADD_ONION (S3.27). See also sections 2.1 and 2.3. */ static std::map<std::string,std::string> ParseTorReplyMapping(const std::string &s) { @@ -272,28 +278,74 @@ static std::map<std::string,std::string> ParseTorReplyMapping(const std::string size_t ptr=0; while (ptr < s.size()) { std::string key, value; - while (ptr < s.size() && s[ptr] != '=') { + while (ptr < s.size() && s[ptr] != '=' && s[ptr] != ' ') { key.push_back(s[ptr]); ++ptr; } if (ptr == s.size()) // unexpected end of line return std::map<std::string,std::string>(); + if (s[ptr] == ' ') // The remaining string is an OptArguments + break; ++ptr; // skip '=' if (ptr < s.size() && s[ptr] == '"') { // Quoted string - ++ptr; // skip '=' + ++ptr; // skip opening '"' bool escape_next = false; - while (ptr < s.size() && (!escape_next && s[ptr] != '"')) { - escape_next = (s[ptr] == '\\'); + while (ptr < s.size() && (escape_next || s[ptr] != '"')) { + // Repeated backslashes must be interpreted as pairs + escape_next = (s[ptr] == '\\' && !escape_next); value.push_back(s[ptr]); ++ptr; } if (ptr == s.size()) // unexpected end of line return std::map<std::string,std::string>(); ++ptr; // skip closing '"' - /* TODO: unescape value - according to the spec this depends on the - * context, some strings use C-LogPrintf style escape codes, some - * don't. So may be better handled at the call site. + /** + * Unescape value. Per https://spec.torproject.org/control-spec section 2.1.1: + * + * For future-proofing, controller implementors MAY use the following + * rules to be compatible with buggy Tor implementations and with + * future ones that implement the spec as intended: + * + * Read \n \t \r and \0 ... \377 as C escapes. + * Treat a backslash followed by any other character as that character. */ + std::string escaped_value; + for (size_t i = 0; i < value.size(); ++i) { + if (value[i] == '\\') { + // This will always be valid, because if the QuotedString + // ended in an odd number of backslashes, then the parser + // would already have returned above, due to a missing + // terminating double-quote. + ++i; + if (value[i] == 'n') { + escaped_value.push_back('\n'); + } else if (value[i] == 't') { + escaped_value.push_back('\t'); + } else if (value[i] == 'r') { + escaped_value.push_back('\r'); + } else if ('0' <= value[i] && value[i] <= '7') { + size_t j; + // Octal escape sequences have a limit of three octal digits, + // but terminate at the first character that is not a valid + // octal digit if encountered sooner. + for (j = 1; j < 3 && (i+j) < value.size() && '0' <= value[i+j] && value[i+j] <= '7'; ++j) {} + // Tor restricts first digit to 0-3 for three-digit octals. + // A leading digit of 4-7 would therefore be interpreted as + // a two-digit octal. + if (j == 3 && value[i] > '3') { + j--; + } + escaped_value.push_back(strtol(value.substr(i, j).c_str(), NULL, 8)); + // Account for automatic incrementing at loop end + i += j - 1; + } else { + escaped_value.push_back(value[i]); + } + } else { + escaped_value.push_back(value[i]); + } + } + value = escaped_value; } else { // Unquoted value. Note that values can contain '=' at will, just no spaces while (ptr < s.size() && s[ptr] != ' ') { value.push_back(s[ptr]); @@ -323,6 +375,10 @@ static std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size char buffer[128]; size_t n; while ((n=fread(buffer, 1, sizeof(buffer), f)) > 0) { + // Check for reading errors so we don't return any data if we couldn't + // read the entire file (or up to maxsize) + if (ferror(f)) + return std::make_pair(false,""); retval.append(buffer, buffer+n); if (retval.size() > maxsize) break; @@ -439,6 +495,13 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe if ((i = m.find("PrivateKey")) != m.end()) private_key = i->second; } + if (service_id.empty()) { + LogPrintf("tor: Error parsing ADD_ONION parameters:\n"); + for (const std::string &s : reply.lines) { + LogPrintf(" %s\n", SanitizeString(s)); + } + return; + } service = LookupNumeric(std::string(service_id+".onion").c_str(), GetListenPort()); LogPrintf("tor: Got service ID %s, advertising service %s\n", service_id, service.ToString()); if (WriteBinaryFile(GetPrivateKeyFile(), private_key)) { @@ -516,6 +579,10 @@ void TorController::authchallenge_cb(TorControlConnection& _conn, const TorContr std::pair<std::string,std::string> l = SplitTorReplyLine(reply.lines[0]); if (l.first == "AUTHCHALLENGE") { std::map<std::string,std::string> m = ParseTorReplyMapping(l.second); + if (m.empty()) { + LogPrintf("tor: Error parsing AUTHCHALLENGE parameters: %s\n", SanitizeString(l.second)); + return; + } std::vector<uint8_t> serverHash = ParseHex(m["SERVERHASH"]); std::vector<uint8_t> serverNonce = ParseHex(m["SERVERNONCE"]); LogPrint(BCLog::TOR, "tor: AUTHCHALLENGE ServerHash %s ServerNonce %s\n", HexStr(serverHash), HexStr(serverNonce)); diff --git a/src/txdb.cpp b/src/txdb.cpp index a3889fdf79..c8f5090293 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -14,6 +14,7 @@ #include <boost/thread.hpp> +static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; @@ -24,17 +25,40 @@ static const char DB_FLAG = 'F'; static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; +namespace { + +struct CoinEntry { + COutPoint* outpoint; + char key; + CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN) {} + + template<typename Stream> + void Serialize(Stream &s) const { + s << key; + s << outpoint->hash; + s << VARINT(outpoint->n); + } + + template<typename Stream> + void Unserialize(Stream& s) { + s >> key; + s >> outpoint->hash; + s >> VARINT(outpoint->n); + } +}; + +} CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) { } -bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const { - return db.Read(std::make_pair(DB_COINS, txid), coins); +bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { + return db.Read(CoinEntry(&outpoint), coin); } -bool CCoinsViewDB::HaveCoins(const uint256 &txid) const { - return db.Exists(std::make_pair(DB_COINS, txid)); +bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { + return db.Exists(CoinEntry(&outpoint)); } uint256 CCoinsViewDB::GetBestBlock() const { @@ -50,10 +74,11 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { size_t changed = 0; for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { - if (it->second.coins.IsPruned()) - batch.Erase(std::make_pair(DB_COINS, it->first)); + CoinEntry entry(&it->first); + if (it->second.coin.IsSpent()) + batch.Erase(entry); else - batch.Write(std::make_pair(DB_COINS, it->first), it->second.coins); + batch.Write(entry, it->second.coin); changed++; } count++; @@ -63,8 +88,14 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { if (!hashBlock.IsNull()) batch.Write(DB_BEST_BLOCK, hashBlock); - LogPrint(BCLog::COINDB, "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); - return db.WriteBatch(batch); + bool ret = db.WriteBatch(batch); + LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count); + return ret; +} + +size_t CCoinsViewDB::EstimateSize() const +{ + return db.EstimateSize(DB_COIN, (char)(DB_COIN+1)); } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { @@ -96,25 +127,31 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const /* It seems that there are no "const iterators" for LevelDB. Since we only need read operations on it, use a const-cast to get around that restriction. */ - i->pcursor->Seek(DB_COINS); + i->pcursor->Seek(DB_COIN); // Cache key of first record - i->pcursor->GetKey(i->keyTmp); + if (i->pcursor->Valid()) { + CoinEntry entry(&i->keyTmp.second); + i->pcursor->GetKey(entry); + i->keyTmp.first = entry.key; + } else { + i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false + } return i; } -bool CCoinsViewDBCursor::GetKey(uint256 &key) const +bool CCoinsViewDBCursor::GetKey(COutPoint &key) const { // Return cached key - if (keyTmp.first == DB_COINS) { + if (keyTmp.first == DB_COIN) { key = keyTmp.second; return true; } return false; } -bool CCoinsViewDBCursor::GetValue(CCoins &coins) const +bool CCoinsViewDBCursor::GetValue(Coin &coin) const { - return pcursor->GetValue(coins); + return pcursor->GetValue(coin); } unsigned int CCoinsViewDBCursor::GetValueSize() const @@ -124,14 +161,18 @@ unsigned int CCoinsViewDBCursor::GetValueSize() const bool CCoinsViewDBCursor::Valid() const { - return keyTmp.first == DB_COINS; + return keyTmp.first == DB_COIN; } void CCoinsViewDBCursor::Next() { pcursor->Next(); - if (!pcursor->Valid() || !pcursor->GetKey(keyTmp)) + CoinEntry entry(&keyTmp.second); + if (!pcursor->Valid() || !pcursor->GetKey(entry)) { keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false + } else { + keyTmp.first = entry.key; + } } bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) { @@ -169,7 +210,7 @@ bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { return true; } -bool CBlockTreeDB::LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256&)> insertBlockIndex) +bool CBlockTreeDB::LoadBlockIndexGuts(std::function<CBlockIndex*(const uint256&)> insertBlockIndex) { std::unique_ptr<CDBIterator> pcursor(NewIterator()); @@ -211,3 +252,103 @@ bool CBlockTreeDB::LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256 return true; } + +namespace { + +//! Legacy class to deserialize pre-pertxout database entries without reindex. +class CCoins +{ +public: + //! whether transaction is a coinbase + bool fCoinBase; + + //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped + std::vector<CTxOut> vout; + + //! at which height this transaction was included in the active block chain + int nHeight; + + //! empty constructor + CCoins() : fCoinBase(false), vout(0), nHeight(0) { } + + template<typename Stream> + void Unserialize(Stream &s) { + unsigned int nCode = 0; + // version + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + // header code + ::Unserialize(s, VARINT(nCode)); + fCoinBase = nCode & 1; + std::vector<bool> vAvail(2, false); + vAvail[0] = (nCode & 2) != 0; + vAvail[1] = (nCode & 4) != 0; + unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); + // spentness bitmask + while (nMaskCode > 0) { + unsigned char chAvail = 0; + ::Unserialize(s, chAvail); + for (unsigned int p = 0; p < 8; p++) { + bool f = (chAvail & (1 << p)) != 0; + vAvail.push_back(f); + } + if (chAvail != 0) + nMaskCode--; + } + // txouts themself + vout.assign(vAvail.size(), CTxOut()); + for (unsigned int i = 0; i < vAvail.size(); i++) { + if (vAvail[i]) + ::Unserialize(s, REF(CTxOutCompressor(vout[i]))); + } + // coinbase height + ::Unserialize(s, VARINT(nHeight)); + } +}; + +} + +/** Upgrade the database from older formats. + * + * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. + */ +bool CCoinsViewDB::Upgrade() { + std::unique_ptr<CDBIterator> pcursor(db.NewIterator()); + pcursor->Seek(std::make_pair(DB_COINS, uint256())); + if (!pcursor->Valid()) { + return true; + } + + LogPrintf("Upgrading database...\n"); + size_t batch_size = 1 << 24; + CDBBatch batch(db); + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair<unsigned char, uint256> key; + if (pcursor->GetKey(key) && key.first == DB_COINS) { + CCoins old_coins; + if (!pcursor->GetValue(old_coins)) { + return error("%s: cannot parse CCoins record", __func__); + } + COutPoint outpoint(key.second, 0); + for (size_t i = 0; i < old_coins.vout.size(); ++i) { + if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) { + Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase); + outpoint.n = i; + CoinEntry entry(&outpoint); + batch.Write(entry, newcoin); + } + } + batch.Erase(key); + if (batch.SizeEstimate() > batch_size) { + db.WriteBatch(batch); + batch.Clear(); + } + pcursor->Next(); + } else { + break; + } + } + db.WriteBatch(batch); + return true; +} diff --git a/src/txdb.h b/src/txdb.h index d9214ba618..974dd4ebe3 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -15,8 +15,6 @@ #include <utility> #include <vector> -#include <boost/function.hpp> - class CBlockIndex; class CCoinsViewDBCursor; class uint256; @@ -24,9 +22,7 @@ class uint256; //! Compensate for extra memory peak (x1.5-x1.9) at flush time. static constexpr int DB_PEAK_USAGE_FACTOR = 2; //! No need to periodic flush if at least this much space still available. -static constexpr int MAX_BLOCK_COINSDB_USAGE = 200 * DB_PEAK_USAGE_FACTOR; -//! Always periodic flush if less than this much space still available. -static constexpr int MIN_BLOCK_COINSDB_USAGE = 50 * DB_PEAK_USAGE_FACTOR; +static constexpr int MAX_BLOCK_COINSDB_USAGE = 10 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; //! max. -dbcache (MiB) @@ -75,11 +71,15 @@ protected: public: CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; - uint256 GetBestBlock() const; - bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock); - CCoinsViewCursor *Cursor() const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const override; + bool HaveCoin(const COutPoint &outpoint) const override; + uint256 GetBestBlock() const override; + bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override; + CCoinsViewCursor *Cursor() const override; + + //! Attempt to update from an older database format. Returns whether an error occurred. + bool Upgrade(); + size_t EstimateSize() const override; }; /** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */ @@ -88,8 +88,8 @@ class CCoinsViewDBCursor: public CCoinsViewCursor public: ~CCoinsViewDBCursor() {} - bool GetKey(uint256 &key) const; - bool GetValue(CCoins &coins) const; + bool GetKey(COutPoint &key) const; + bool GetValue(Coin &coin) const; unsigned int GetValueSize() const; bool Valid() const; @@ -99,7 +99,7 @@ private: CCoinsViewDBCursor(CDBIterator* pcursorIn, const uint256 &hashBlockIn): CCoinsViewCursor(hashBlockIn), pcursor(pcursorIn) {} std::unique_ptr<CDBIterator> pcursor; - std::pair<char, uint256> keyTmp; + std::pair<char, COutPoint> keyTmp; friend class CCoinsViewDB; }; @@ -122,7 +122,7 @@ public: bool WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> > &list); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); - bool LoadBlockIndexGuts(boost::function<CBlockIndex*(const uint256&)> insertBlockIndex); + bool LoadBlockIndexGuts(std::function<CBlockIndex*(const uint256&)> insertBlockIndex); }; #endif // BITCOIN_TXDB_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 0794a3902f..17389db9f0 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -5,8 +5,8 @@ #include "txmempool.h" -#include "clientversion.h" #include "consensus/consensus.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "validation.h" #include "policy/policy.h" @@ -16,7 +16,6 @@ #include "util.h" #include "utilmoneystr.h" #include "utiltime.h" -#include "version.h" CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, int64_t _nTime, unsigned int _entryHeight, @@ -25,7 +24,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFe spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { nTxWeight = GetTransactionWeight(*tx); - nUsageSize = RecursiveDynamicUsage(*tx) + memusage::DynamicUsage(tx); + nUsageSize = RecursiveDynamicUsage(tx); nCountWithDescendants = 1; nSizeWithDescendants = GetTxSize(); @@ -333,8 +332,8 @@ void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, assert(int(nSigOpCostWithAncestors) >= 0); } -CTxMemPool::CTxMemPool() : - nTransactionsUpdated(0) +CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) : + nTransactionsUpdated(0), minerPolicyEstimator(estimator) { _clear(); //lock free clear @@ -342,26 +341,12 @@ CTxMemPool::CTxMemPool() : // accepting transactions becomes O(N^2) where N is the number // of transactions in the pool nCheckFrequency = 0; - - minerPolicyEstimator = new CBlockPolicyEstimator(); } -CTxMemPool::~CTxMemPool() -{ - delete minerPolicyEstimator; -} - -void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins) +bool CTxMemPool::isSpent(const COutPoint& outpoint) { LOCK(cs); - - auto it = mapNextTx.lower_bound(COutPoint(hashTx, 0)); - - // iterate over all COutPoints in mapNextTx whose hash equals the provided hashTx - while (it != mapNextTx.end() && it->first->hash == hashTx) { - coins.Spend(it->first->n); // and remove those outputs from coins - it++; - } + return mapNextTx.count(outpoint); } unsigned int CTxMemPool::GetTransactionsUpdated() const @@ -427,7 +412,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - minerPolicyEstimator->processTransaction(entry, validFeeEstimate); + if (minerPolicyEstimator) {minerPolicyEstimator->processTransaction(entry, validFeeEstimate);} vTxHashes.emplace_back(tx.GetWitnessHash(), newit); newit->vTxHashesIdx = vTxHashes.size() - 1; @@ -457,7 +442,7 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) mapLinks.erase(it); mapTx.erase(it); nTransactionsUpdated++; - minerPolicyEstimator->removeTx(hash); + if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash, false);} } // Calculates descendants of entry that are not already in setDescendants, and adds to @@ -539,9 +524,9 @@ void CTxMemPool::removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMem indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; - const CCoins *coins = pcoins->AccessCoins(txin.prevout.hash); - if (nCheckFrequency != 0) assert(coins); - if (!coins || (coins->IsCoinBase() && ((signed long)nMemPoolHeight) - coins->nHeight < COINBASE_MATURITY)) { + const Coin &coin = pcoins->AccessCoin(txin.prevout); + if (nCheckFrequency != 0) assert(!coin.IsSpent()); + if (coin.IsSpent() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < COINBASE_MATURITY)) { txToRemove.insert(it); break; } @@ -591,7 +576,7 @@ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigne entries.push_back(&*i); } // Before the txs in the new block have been removed from the mempool, update policy estimates - minerPolicyEstimator->processBlock(nBlockHeight, entries); + if (minerPolicyEstimator) {minerPolicyEstimator->processBlock(nBlockHeight, entries);} for (const auto& tx : vtx) { txiter it = mapTx.find(tx->GetHash()); @@ -669,8 +654,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const parentSigOpCost += it2->GetSigOpCost(); } } else { - const CCoins* coins = pcoins->AccessCoins(txin.prevout.hash); - assert(coins && coins->IsAvailable(txin.prevout.n)); + assert(pcoins->HaveCoin(txin.prevout)); } // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); @@ -850,51 +834,6 @@ TxMempoolInfo CTxMemPool::info(const uint256& hash) const return GetInfo(i); } -CFeeRate CTxMemPool::estimateFee(int nBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateFee(nBlocks); -} -CFeeRate CTxMemPool::estimateSmartFee(int nBlocks, int *answerFoundAtBlocks) const -{ - LOCK(cs); - return minerPolicyEstimator->estimateSmartFee(nBlocks, answerFoundAtBlocks, *this); -} - -bool -CTxMemPool::WriteFeeEstimates(CAutoFile& fileout) const -{ - try { - LOCK(cs); - fileout << 139900; // version required to read: 0.13.99 or later - fileout << CLIENT_VERSION; // version that wrote the file - minerPolicyEstimator->Write(fileout); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::WriteFeeEstimates(): unable to write policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - -bool -CTxMemPool::ReadFeeEstimates(CAutoFile& filein) -{ - try { - int nVersionRequired, nVersionThatWrote; - filein >> nVersionRequired >> nVersionThatWrote; - if (nVersionRequired > CLIENT_VERSION) - return error("CTxMemPool::ReadFeeEstimates(): up-version (%d) fee estimate file", nVersionRequired); - LOCK(cs); - minerPolicyEstimator->Read(filein, nVersionThatWrote); - } - catch (const std::exception&) { - LogPrintf("CTxMemPool::ReadFeeEstimates(): unable to read policy estimator data (non-fatal)\n"); - return false; - } - return true; -} - void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { @@ -919,6 +858,7 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD BOOST_FOREACH(txiter descendantIt, setDescendants) { mapTx.modify(descendantIt, update_ancestor_state(0, nFeeDelta, 0, 0)); } + ++nTransactionsUpdated; } } LogPrintf("PrioritiseTransaction: %s feerate += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); @@ -950,20 +890,24 @@ bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { } -bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) const { +bool CCoinsViewMemPool::GetCoin(const COutPoint &outpoint, Coin &coin) const { // If an entry in the mempool exists, always return that one, as it's guaranteed to never // conflict with the underlying cache, and it cannot have pruned entries (as it contains full) // transactions. First checking the underlying cache risks returning a pruned entry instead. - CTransactionRef ptx = mempool.get(txid); + CTransactionRef ptx = mempool.get(outpoint.hash); if (ptx) { - coins = CCoins(*ptx, MEMPOOL_HEIGHT); - return true; + if (outpoint.n < ptx->vout.size()) { + coin = Coin(ptx->vout[outpoint.n], MEMPOOL_HEIGHT, false); + return true; + } else { + return false; + } } - return (base->GetCoins(txid, coins) && !coins.IsPruned()); + return (base->GetCoin(outpoint, coin) && !coin.IsSpent()); } -bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const { - return mempool.exists(txid) || base->HaveCoins(txid); +bool CCoinsViewMemPool::HaveCoin(const COutPoint &outpoint) const { + return mempool.exists(outpoint) || base->HaveCoin(outpoint); } size_t CTxMemPool::DynamicMemoryUsage() const { @@ -1074,7 +1018,7 @@ void CTxMemPool::trackPackageRemoved(const CFeeRate& rate) { } } -void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRemaining) { +void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpendsRemaining) { LOCK(cs); unsigned nTxnRemoved = 0; @@ -1105,11 +1049,10 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRe if (pvNoSpendsRemaining) { BOOST_FOREACH(const CTransaction& tx, txn) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { - if (exists(txin.prevout.hash)) - continue; - auto iter = mapNextTx.lower_bound(COutPoint(txin.prevout.hash, 0)); - if (iter == mapNextTx.end() || iter->first->hash != txin.prevout.hash) - pvNoSpendsRemaining->push_back(txin.prevout.hash); + if (exists(txin.prevout.hash)) continue; + if (!mapNextTx.count(txin.prevout)) { + pvNoSpendsRemaining->push_back(txin.prevout); + } } } } @@ -1126,3 +1069,5 @@ bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLi return it == mapTx.end() || (it->GetCountWithAncestors() < chainLimit && it->GetCountWithDescendants() < chainLimit); } + +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index 4222789510..0316b42ba2 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -16,6 +16,7 @@ #include "amount.h" #include "coins.h" #include "indirectmap.h" +#include "policy/feerate.h" #include "primitives/transaction.h" #include "sync.h" #include "random.h" @@ -23,14 +24,15 @@ #include "boost/multi_index_container.hpp" #include "boost/multi_index/ordered_index.hpp" #include "boost/multi_index/hashed_index.hpp" +#include <boost/multi_index/sequenced_index.hpp> #include <boost/signals2/signal.hpp> class CAutoFile; class CBlockIndex; -/** Fake height value used in CCoins to signify they are only in the memory pool (since 0.8) */ -static const unsigned int MEMPOOL_HEIGHT = 0x7FFFFFFF; +/** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ +static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; struct LockPoints { @@ -59,11 +61,6 @@ class CTxMemPool; * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) for * all ancestors of the newly added transaction. * - * If updating the descendant state is skipped, we can mark the entry as - * "dirty", and set nSizeWithDescendants/nModFeesWithDescendants to equal nTxSize/ - * nFee+feeDelta. (This can potentially happen during a reorg, where we limit the - * amount of work we're willing to do to avoid consuming too much CPU.) - * */ class CTxMemPoolEntry @@ -82,9 +79,7 @@ private: // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these - // descendants as well. if nCountWithDescendants is 0, treat this entry as - // dirty, and nSizeWithDescendants and nModFeesWithDescendants will not be - // correct. + // descendants as well. uint64_t nCountWithDescendants; //!< number of descendant transactions uint64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) @@ -115,7 +110,7 @@ public: size_t DynamicMemoryUsage() const { return nUsageSize; } const LockPoints& GetLockPoints() const { return lockPoints; } - // Adjusts the descendant state, if this entry is not dirty. + // Adjusts the descendant state. void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps); @@ -191,7 +186,7 @@ private: const LockPoints& lp; }; -// extracts a TxMemPoolEntry's transaction hash +// extracts a transaction hash from CTxMempoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; @@ -199,6 +194,11 @@ struct mempoolentry_txid { return entry.GetTx().GetHash(); } + + result_type operator() (const CTransactionRef& tx) const + { + return tx->GetHash(); + } }; /** \class CompareTxMemPoolEntryByDescendantScore @@ -327,6 +327,20 @@ enum class MemPoolRemovalReason { REPLACED //! Removed for replacement }; +class SaltedTxidHasher +{ +private: + /** Salt */ + const uint64_t k0, k1; + +public: + SaltedTxidHasher(); + + size_t operator()(const uint256& txid) const { + return SipHashUint256(k0, k1, txid); + } +}; + /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions * that may be included in the next block. @@ -398,20 +412,12 @@ enum class MemPoolRemovalReason { * CalculateMemPoolAncestors() takes configurable limits that are designed to * prevent these calculations from being too CPU intensive. * - * Adding transactions from a disconnected block can be very time consuming, - * because we don't have a way to limit the number of in-mempool descendants. - * To bound CPU processing, we limit the amount of work we're willing to do - * to properly update the descendant information for a tx being added from - * a disconnected block. If we would exceed the limit, then we instead mark - * the entry as "dirty", and set the feerate for sorting purposes to be equal - * the feerate of the transaction without any descendants. - * */ class CTxMemPool { private: uint32_t nCheckFrequency; //!< Value n means that n times in 2^32 we check. - unsigned int nTransactionsUpdated; + unsigned int nTransactionsUpdated; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation CBlockPolicyEstimator* minerPolicyEstimator; uint64_t totalTxSize; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. @@ -496,8 +502,7 @@ public: /** Create a new CTxMemPool. */ - CTxMemPool(); - ~CTxMemPool(); + CTxMemPool(CBlockPolicyEstimator* estimator = nullptr); /** * If sanity-checking is turned on, check makes sure the pool is @@ -524,7 +529,7 @@ public: void _clear(); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); void queryHashes(std::vector<uint256>& vtxid); - void pruneSpent(const uint256& hash, CCoins &coins); + bool isSpent(const COutPoint& outpoint); unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); /** @@ -585,10 +590,10 @@ public: CFeeRate GetMinFee(size_t sizelimit) const; /** Remove transactions from the mempool until its dynamic size is <= sizelimit. - * pvNoSpendsRemaining, if set, will be populated with the list of transactions + * pvNoSpendsRemaining, if set, will be populated with the list of outpoints * which are not in mempool which no longer have any spends in this mempool. */ - void TrimToSize(size_t sizelimit, std::vector<uint256>* pvNoSpendsRemaining=NULL); + void TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpendsRemaining=NULL); /** Expire all transaction (and their dependencies) in the mempool older than time. Return the number of removed transactions. */ int Expire(int64_t time); @@ -614,23 +619,17 @@ public: return (mapTx.count(hash) != 0); } + bool exists(const COutPoint& outpoint) const + { + LOCK(cs); + auto it = mapTx.find(outpoint.hash); + return (it != mapTx.end() && outpoint.n < it->GetTx().vout.size()); + } + CTransactionRef get(const uint256& hash) const; TxMempoolInfo info(const uint256& hash) const; std::vector<TxMempoolInfo> infoAll() const; - /** Estimate fee rate needed to get into the next nBlocks - * If no answer can be given at nBlocks, return an estimate - * at the lowest number of blocks where one can be given - */ - CFeeRate estimateSmartFee(int nBlocks, int *answerFoundAtBlocks = NULL) const; - - /** Estimate fee rate needed to get into the next nBlocks */ - CFeeRate estimateFee(int nBlocks) const; - - /** Write/Read estimates to disk */ - bool WriteFeeEstimates(CAutoFile& fileout) const; - bool ReadFeeEstimates(CAutoFile& filein); - size_t DynamicMemoryUsage() const; boost::signals2::signal<void (CTransactionRef)> NotifyEntryAdded; @@ -686,8 +685,99 @@ protected: public: CCoinsViewMemPool(CCoinsView* baseIn, const CTxMemPool& mempoolIn); - bool GetCoins(const uint256 &txid, CCoins &coins) const; - bool HaveCoins(const uint256 &txid) const; + bool GetCoin(const COutPoint &outpoint, Coin &coin) const; + bool HaveCoin(const COutPoint &outpoint) const; +}; + +/** + * DisconnectedBlockTransactions + + * During the reorg, it's desirable to re-add previously confirmed transactions + * to the mempool, so that anything not re-confirmed in the new chain is + * available to be mined. However, it's more efficient to wait until the reorg + * is complete and process all still-unconfirmed transactions at that time, + * since we expect most confirmed transactions to (typically) still be + * confirmed in the new chain, and re-accepting to the memory pool is expensive + * (and therefore better to not do in the middle of reorg-processing). + * Instead, store the disconnected transactions (in order!) as we go, remove any + * that are included in blocks in the new chain, and then process the remaining + * still-unconfirmed transactions at the end. + */ + +// multi_index tag names +struct txid_index {}; +struct insertion_order {}; + +struct DisconnectedBlockTransactions { + typedef boost::multi_index_container< + CTransactionRef, + boost::multi_index::indexed_by< + // sorted by txid + boost::multi_index::hashed_unique< + boost::multi_index::tag<txid_index>, + mempoolentry_txid, + SaltedTxidHasher + >, + // sorted by order in the blockchain + boost::multi_index::sequenced< + boost::multi_index::tag<insertion_order> + > + > + > indexed_disconnected_transactions; + + // It's almost certainly a logic bug if we don't clear out queuedTx before + // destruction, as we add to it while disconnecting blocks, and then we + // need to re-process remaining transactions to ensure mempool consistency. + // For now, assert() that we've emptied out this object on destruction. + // This assert() can always be removed if the reorg-processing code were + // to be refactored such that this assumption is no longer true (for + // instance if there was some other way we cleaned up the mempool after a + // reorg, besides draining this object). + ~DisconnectedBlockTransactions() { assert(queuedTx.empty()); } + + indexed_disconnected_transactions queuedTx; + uint64_t cachedInnerUsage = 0; + + // Estimate the overhead of queuedTx to be 6 pointers + an allocation, as + // no exact formula for boost::multi_index_contained is implemented. + size_t DynamicMemoryUsage() const { + return memusage::MallocUsage(sizeof(CTransactionRef) + 6 * sizeof(void*)) * queuedTx.size() + cachedInnerUsage; + } + + void addTransaction(const CTransactionRef& tx) + { + queuedTx.insert(tx); + cachedInnerUsage += RecursiveDynamicUsage(tx); + } + + // Remove entries based on txid_index, and update memory usage. + void removeForBlock(const std::vector<CTransactionRef>& vtx) + { + // Short-circuit in the common case of a block being added to the tip + if (queuedTx.empty()) { + return; + } + for (auto const &tx : vtx) { + auto it = queuedTx.find(tx->GetHash()); + if (it != queuedTx.end()) { + cachedInnerUsage -= RecursiveDynamicUsage(*it); + queuedTx.erase(it); + } + } + } + + // Remove an entry by insertion_order index, and update memory usage. + void removeEntry(indexed_disconnected_transactions::index<insertion_order>::type::iterator entry) + { + cachedInnerUsage -= RecursiveDynamicUsage(*entry); + queuedTx.get<insertion_order>().erase(entry); + } + + void clear() + { + cachedInnerUsage = 0; + queuedTx.clear(); + } }; #endif // BITCOIN_TXMEMPOOL_H diff --git a/src/undo.h b/src/undo.h index a94e31f5cc..3749d5d7a8 100644 --- a/src/undo.h +++ b/src/undo.h @@ -7,58 +7,90 @@ #define BITCOIN_UNDO_H #include "compressor.h" +#include "consensus/consensus.h" #include "primitives/transaction.h" #include "serialize.h" /** Undo information for a CTxIn * - * Contains the prevout's CTxOut being spent, and if this was the - * last output of the affected transaction, its metadata as well - * (coinbase or not, height, transaction version) + * Contains the prevout's CTxOut being spent, and its metadata as well + * (coinbase or not, height). The serialization contains a dummy value of + * zero. This is be compatible with older versions which expect to see + * the transaction version there. */ -class CTxInUndo +class TxInUndoSerializer { -public: - CTxOut txout; // the txout data before being spent - bool fCoinBase; // if the outpoint was the last unspent: whether it belonged to a coinbase - unsigned int nHeight; // if the outpoint was the last unspent: its height - int nVersion; // if the outpoint was the last unspent: its version - - CTxInUndo() : txout(), fCoinBase(false), nHeight(0), nVersion(0) {} - CTxInUndo(const CTxOut &txoutIn, bool fCoinBaseIn = false, unsigned int nHeightIn = 0, int nVersionIn = 0) : txout(txoutIn), fCoinBase(fCoinBaseIn), nHeight(nHeightIn), nVersion(nVersionIn) { } + const Coin* txout; +public: template<typename Stream> void Serialize(Stream &s) const { - ::Serialize(s, VARINT(nHeight*2+(fCoinBase ? 1 : 0))); - if (nHeight > 0) - ::Serialize(s, VARINT(this->nVersion)); - ::Serialize(s, CTxOutCompressor(REF(txout))); + ::Serialize(s, VARINT(txout->nHeight * 2 + (txout->fCoinBase ? 1 : 0))); + if (txout->nHeight > 0) { + // Required to maintain compatibility with older undo format. + ::Serialize(s, (unsigned char)0); + } + ::Serialize(s, CTxOutCompressor(REF(txout->out))); } + TxInUndoSerializer(const Coin* coin) : txout(coin) {} +}; + +class TxInUndoDeserializer +{ + Coin* txout; + +public: template<typename Stream> void Unserialize(Stream &s) { unsigned int nCode = 0; ::Unserialize(s, VARINT(nCode)); - nHeight = nCode / 2; - fCoinBase = nCode & 1; - if (nHeight > 0) - ::Unserialize(s, VARINT(this->nVersion)); - ::Unserialize(s, REF(CTxOutCompressor(REF(txout)))); + txout->nHeight = nCode / 2; + txout->fCoinBase = nCode & 1; + if (txout->nHeight > 0) { + // Old versions stored the version number for the last spend of + // a transaction's outputs. Non-final spends were indicated with + // height = 0. + int nVersionDummy; + ::Unserialize(s, VARINT(nVersionDummy)); + } + ::Unserialize(s, REF(CTxOutCompressor(REF(txout->out)))); } + + TxInUndoDeserializer(Coin* coin) : txout(coin) {} }; +static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_BASE_SIZE / ::GetSerializeSize(CTxIn(), SER_NETWORK, PROTOCOL_VERSION); + /** Undo information for a CTransaction */ class CTxUndo { public: // undo information for all txins - std::vector<CTxInUndo> vprevout; + std::vector<Coin> vprevout; - ADD_SERIALIZE_METHODS; + template <typename Stream> + void Serialize(Stream& s) const { + // TODO: avoid reimplementing vector serializer + uint64_t count = vprevout.size(); + ::Serialize(s, COMPACTSIZE(REF(count))); + for (const auto& prevout : vprevout) { + ::Serialize(s, REF(TxInUndoSerializer(&prevout))); + } + } - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vprevout); + template <typename Stream> + void Unserialize(Stream& s) { + // TODO: avoid reimplementing vector deserializer + uint64_t count = 0; + ::Unserialize(s, COMPACTSIZE(count)); + if (count > MAX_INPUTS_PER_BLOCK) { + throw std::ios_base::failure("Too many input undo records"); + } + vprevout.resize(count); + for (auto& prevout : vprevout) { + ::Unserialize(s, REF(TxInUndoDeserializer(&prevout))); + } } }; diff --git a/src/util.cpp b/src/util.cpp index 0dc203cba5..653a4f072a 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -13,7 +13,6 @@ #include "fs.h" #include "random.h" #include "serialize.h" -#include "sync.h" #include "utilstrencodings.h" #include "utiltime.h" @@ -88,27 +87,11 @@ #include <openssl/rand.h> #include <openssl/conf.h> -// Work around clang compilation problem in Boost 1.46: -// /usr/include/boost/program_options/detail/config_file.hpp:163:17: error: call to function 'to_internal' that is neither visible in the template definition nor found by argument-dependent lookup -// See also: http://stackoverflow.com/questions/10020179/compilation-fail-in-boost-librairies-program-options -// http://clang.debian.net/status.php?version=3.0&key=CANNOT_FIND_FUNCTION -namespace boost { - - namespace program_options { - std::string to_internal(const std::string&); - } - -} // namespace boost - - const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char * const BITCOIN_PID_FILENAME = "bitcoind.pid"; -CCriticalSection cs_args; -std::map<std::string, std::string> mapArgs; -static std::map<std::string, std::vector<std::string> > _mapMultiArgs; -const std::map<std::string, std::vector<std::string> >& mapMultiArgs = _mapMultiArgs; +ArgsManager gArgs; bool fPrintToConsole = false; bool fPrintToDebugLog = true; @@ -323,10 +306,14 @@ static std::string LogTimestampStr(const std::string &str, std::atomic_bool *fSt return str; if (*fStartedNewLine) { - int64_t nTimeMicros = GetLogTimeMicros(); + int64_t nTimeMicros = GetTimeMicros(); strStamped = DateTimeStrFormat("%Y-%m-%d %H:%M:%S", nTimeMicros/1000000); if (fLogTimeMicros) strStamped += strprintf(".%06d", nTimeMicros%1000000); + int64_t mocktime = GetMockTime(); + if (mocktime) { + strStamped += " (mocktime: " + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", mocktime) + ")"; + } strStamped += ' ' + str; } else strStamped = str; @@ -397,11 +384,11 @@ static void InterpretNegativeSetting(std::string& strKey, std::string& strValue) } } -void ParseParameters(int argc, const char* const argv[]) +void ArgsManager::ParseParameters(int argc, const char* const argv[]) { LOCK(cs_args); mapArgs.clear(); - _mapMultiArgs.clear(); + mapMultiArgs.clear(); for (int i = 1; i < argc; i++) { @@ -429,17 +416,23 @@ void ParseParameters(int argc, const char* const argv[]) InterpretNegativeSetting(str, strValue); mapArgs[str] = strValue; - _mapMultiArgs[str].push_back(strValue); + mapMultiArgs[str].push_back(strValue); } } -bool IsArgSet(const std::string& strArg) +std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) +{ + LOCK(cs_args); + return mapMultiArgs.at(strArg); +} + +bool ArgsManager::IsArgSet(const std::string& strArg) { LOCK(cs_args); return mapArgs.count(strArg); } -std::string GetArg(const std::string& strArg, const std::string& strDefault) +std::string ArgsManager::GetArg(const std::string& strArg, const std::string& strDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -447,7 +440,7 @@ std::string GetArg(const std::string& strArg, const std::string& strDefault) return strDefault; } -int64_t GetArg(const std::string& strArg, int64_t nDefault) +int64_t ArgsManager::GetArg(const std::string& strArg, int64_t nDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -455,7 +448,7 @@ int64_t GetArg(const std::string& strArg, int64_t nDefault) return nDefault; } -bool GetBoolArg(const std::string& strArg, bool fDefault) +bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) { LOCK(cs_args); if (mapArgs.count(strArg)) @@ -463,16 +456,16 @@ bool GetBoolArg(const std::string& strArg, bool fDefault) return fDefault; } -bool SoftSetArg(const std::string& strArg, const std::string& strValue) +bool ArgsManager::SoftSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); if (mapArgs.count(strArg)) return false; - mapArgs[strArg] = strValue; + ForceSetArg(strArg, strValue); return true; } -bool SoftSetBoolArg(const std::string& strArg, bool fValue) +bool ArgsManager::SoftSetBoolArg(const std::string& strArg, bool fValue) { if (fValue) return SoftSetArg(strArg, std::string("1")); @@ -480,10 +473,11 @@ bool SoftSetBoolArg(const std::string& strArg, bool fValue) return SoftSetArg(strArg, std::string("0")); } -void ForceSetArg(const std::string& strArg, const std::string& strValue) +void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); mapArgs[strArg] = strValue; + mapMultiArgs[strArg].push_back(strValue); } @@ -602,7 +596,7 @@ fs::path GetConfigFile(const std::string& confPath) return pathConfigFile; } -void ReadConfigFile(const std::string& confPath) +void ArgsManager::ReadConfigFile(const std::string& confPath) { fs::ifstream streamConfig(GetConfigFile(confPath)); if (!streamConfig.good()) @@ -621,7 +615,7 @@ void ReadConfigFile(const std::string& confPath) InterpretNegativeSetting(strKey, strValue); if (mapArgs.count(strKey) == 0) mapArgs[strKey] = strValue; - _mapMultiArgs[strKey].push_back(strValue); + mapMultiArgs[strKey].push_back(strValue); } } // If datadir is changed in .conf file: diff --git a/src/util.h b/src/util.h index ed28070a3f..4386ddd550 100644 --- a/src/util.h +++ b/src/util.h @@ -16,6 +16,7 @@ #include "compat.h" #include "fs.h" +#include "sync.h" #include "tinyformat.h" #include "utiltime.h" @@ -41,7 +42,6 @@ public: boost::signals2::signal<std::string (const char* psz)> Translate; }; -extern const std::map<std::string, std::vector<std::string> >& mapMultiArgs; extern bool fPrintToConsole; extern bool fPrintToDebugLog; @@ -148,7 +148,6 @@ bool error(const char* fmt, const Args&... args) } void PrintExceptionContinue(const std::exception *pex, const char* pszThread); -void ParseParameters(int argc, const char*const argv[]); void FileCommit(FILE *file); bool TruncateFile(FILE *file, unsigned int length); int RaiseFileDescriptorLimit(int nMinFD); @@ -163,7 +162,6 @@ fs::path GetConfigFile(const std::string& confPath); fs::path GetPidFile(); void CreatePidFile(const fs::path &path, pid_t pid); #endif -void ReadConfigFile(const std::string& confPath); #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif @@ -180,6 +178,16 @@ inline bool IsSwitchChar(char c) #endif } +class ArgsManager +{ +protected: + CCriticalSection cs_args; + std::map<std::string, std::string> mapArgs; + std::map<std::string, std::vector<std::string> > mapMultiArgs; +public: + void ParseParameters(int argc, const char*const argv[]); + void ReadConfigFile(const std::string& confPath); + std::vector<std::string> GetArgs(const std::string& strArg); /** * Return true if the given argument has been manually set * @@ -233,8 +241,58 @@ bool SoftSetArg(const std::string& strArg, const std::string& strValue); */ bool SoftSetBoolArg(const std::string& strArg, bool fValue); -// Forces a arg setting, used only in testing +// Forces an arg setting. Called by SoftSetArg() if the arg hasn't already +// been set. Also called directly in testing. void ForceSetArg(const std::string& strArg, const std::string& strValue); +}; + +extern ArgsManager gArgs; + +// wrappers using the global ArgsManager: +static inline void ParseParameters(int argc, const char*const argv[]) +{ + gArgs.ParseParameters(argc, argv); +} + +static inline void ReadConfigFile(const std::string& confPath) +{ + gArgs.ReadConfigFile(confPath); +} + +static inline bool SoftSetArg(const std::string& strArg, const std::string& strValue) +{ + return gArgs.SoftSetArg(strArg, strValue); +} + +static inline void ForceSetArg(const std::string& strArg, const std::string& strValue) +{ + gArgs.ForceSetArg(strArg, strValue); +} + +static inline bool IsArgSet(const std::string& strArg) +{ + return gArgs.IsArgSet(strArg); +} + +static inline std::string GetArg(const std::string& strArg, const std::string& strDefault) +{ + return gArgs.GetArg(strArg, strDefault); +} + +static inline int64_t GetArg(const std::string& strArg, int64_t nDefault) +{ + return gArgs.GetArg(strArg, nDefault); +} + +static inline bool GetBoolArg(const std::string& strArg, bool fDefault) +{ + return gArgs.GetBoolArg(strArg, fDefault); +} + +static inline bool SoftSetBoolArg(const std::string& strArg, bool fValue) +{ + return gArgs.SoftSetBoolArg(strArg, fValue); +} /** * Format a string to be used as group of options in help messages diff --git a/src/utiltime.cpp b/src/utiltime.cpp index a9936a645a..e07069125d 100644 --- a/src/utiltime.cpp +++ b/src/utiltime.cpp @@ -9,14 +9,17 @@ #include "utiltime.h" +#include <atomic> + #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/thread.hpp> -static int64_t nMockTime = 0; //!< For unit testing +static std::atomic<int64_t> nMockTime(0); //!< For unit testing int64_t GetTime() { - if (nMockTime) return nMockTime; + int64_t mocktime = nMockTime.load(std::memory_order_relaxed); + if (mocktime) return mocktime; time_t now = time(NULL); assert(now > 0); @@ -25,7 +28,12 @@ int64_t GetTime() void SetMockTime(int64_t nMockTimeIn) { - nMockTime = nMockTimeIn; + nMockTime.store(nMockTimeIn, std::memory_order_relaxed); +} + +int64_t GetMockTime() +{ + return nMockTime.load(std::memory_order_relaxed); } int64_t GetTimeMillis() @@ -49,14 +57,6 @@ int64_t GetSystemTimeInSeconds() return GetTimeMicros()/1000000; } -/** Return a time useful for the debug log */ -int64_t GetLogTimeMicros() -{ - if (nMockTime) return nMockTime*1000000; - - return GetTimeMicros(); -} - void MilliSleep(int64_t n) { diff --git a/src/utiltime.h b/src/utiltime.h index cc3290c631..8ae8540b89 100644 --- a/src/utiltime.h +++ b/src/utiltime.h @@ -23,8 +23,8 @@ int64_t GetTime(); int64_t GetTimeMillis(); int64_t GetTimeMicros(); int64_t GetSystemTimeInSeconds(); // Like GetTime(), but not mockable -int64_t GetLogTimeMicros(); void SetMockTime(int64_t nMockTimeIn); +int64_t GetMockTime(); void MilliSleep(int64_t n); std::string DateTimeStrFormat(const char* pszFormat, int64_t nTime); diff --git a/src/validation.cpp b/src/validation.cpp index 99ce53986f..de65839eef 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -6,11 +6,13 @@ #include "validation.h" #include "arith_uint256.h" +#include "chain.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" #include "consensus/consensus.h" #include "consensus/merkle.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "fs.h" #include "hash.h" @@ -80,7 +82,8 @@ uint256 hashAssumeValid; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; -CTxMemPool mempool; +CBlockPolicyEstimator feeEstimator; +CTxMemPool mempool(&feeEstimator); static void CheckBlockIndex(const Consensus::Params& consensusParams); @@ -186,19 +189,6 @@ enum FlushStateMode { bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int nManualPruneHeight=0); void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); -bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) -{ - if (tx.nLockTime == 0) - return true; - if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) - return true; - for (const auto& txin : tx.vin) { - if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) - return false; - } - return true; -} - bool CheckFinalTx(const CTransaction &tx, int flags) { AssertLockHeld(cs_main); @@ -231,89 +221,6 @@ bool CheckFinalTx(const CTransaction &tx, int flags) return IsFinalTx(tx, nBlockHeight, nBlockTime); } -/** - * Calculates the block height and previous block's median time past at - * which the transaction will be considered final in the context of BIP 68. - * Also removes from the vector of input heights any entries which did not - * correspond to sequence locked inputs as they do not affect the calculation. - */ -static std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) -{ - assert(prevHeights->size() == tx.vin.size()); - - // Will be set to the equivalent height- and time-based nLockTime - // values that would be necessary to satisfy all relative lock- - // time constraints given our view of block chain history. - // The semantics of nLockTime are the last invalid height/time, so - // use -1 to have the effect of any height or time being valid. - int nMinHeight = -1; - int64_t nMinTime = -1; - - // tx.nVersion is signed integer so requires cast to unsigned otherwise - // we would be doing a signed comparison and half the range of nVersion - // wouldn't support BIP 68. - bool fEnforceBIP68 = static_cast<uint32_t>(tx.nVersion) >= 2 - && flags & LOCKTIME_VERIFY_SEQUENCE; - - // Do not enforce sequence numbers as a relative lock time - // unless we have been instructed to - if (!fEnforceBIP68) { - return std::make_pair(nMinHeight, nMinTime); - } - - for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { - const CTxIn& txin = tx.vin[txinIndex]; - - // Sequence numbers with the most significant bit set are not - // treated as relative lock-times, nor are they given any - // consensus-enforced meaning at this point. - if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_DISABLE_FLAG) { - // The height of this input is not relevant for sequence locks - (*prevHeights)[txinIndex] = 0; - continue; - } - - int nCoinHeight = (*prevHeights)[txinIndex]; - - if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) { - int64_t nCoinTime = block.GetAncestor(std::max(nCoinHeight-1, 0))->GetMedianTimePast(); - // NOTE: Subtract 1 to maintain nLockTime semantics - // BIP 68 relative lock times have the semantics of calculating - // the first block or time at which the transaction would be - // valid. When calculating the effective block time or height - // for the entire transaction, we switch to using the - // semantics of nLockTime which is the last invalid block - // time or height. Thus we subtract 1 from the calculated - // time or height. - - // Time-based relative lock-times are measured from the - // smallest allowed timestamp of the block containing the - // txout being spent, which is the median time past of the - // block prior. - nMinTime = std::max(nMinTime, nCoinTime + (int64_t)((txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) << CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) - 1); - } else { - nMinHeight = std::max(nMinHeight, nCoinHeight + (int)(txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) - 1); - } - } - - return std::make_pair(nMinHeight, nMinTime); -} - -static bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair) -{ - assert(block.pprev); - int64_t nBlockTime = block.pprev->GetMedianTimePast(); - if (lockPair.first >= block.nHeight || lockPair.second >= nBlockTime) - return false; - - return true; -} - -bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) -{ - return EvaluateSequenceLocks(block, CalculateSequenceLocks(tx, flags, prevHeights, block)); -} - bool TestLockPointValidity(const LockPoints* lp) { AssertLockHeld(cs_main); @@ -361,15 +268,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool prevheights.resize(tx.vin.size()); for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { const CTxIn& txin = tx.vin[txinIndex]; - CCoins coins; - if (!viewMemPool.GetCoins(txin.prevout.hash, coins)) { + Coin coin; + if (!viewMemPool.GetCoin(txin.prevout, coin)) { return error("%s: Missing input", __func__); } - if (coins.nHeight == MEMPOOL_HEIGHT) { + if (coin.nHeight == MEMPOOL_HEIGHT) { // Assume all mempool transaction confirm in the next block prevheights[txinIndex] = tip->nHeight + 1; } else { - prevheights[txinIndex] = coins.nHeight; + prevheights[txinIndex] = coin.nHeight; } } lockPair = CalculateSequenceLocks(tx, flags, &prevheights, index); @@ -402,117 +309,15 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool return EvaluateSequenceLocks(index, lockPair); } - -unsigned int GetLegacySigOpCount(const CTransaction& tx) -{ - unsigned int nSigOps = 0; - for (const auto& txin : tx.vin) - { - nSigOps += txin.scriptSig.GetSigOpCount(false); - } - for (const auto& txout : tx.vout) - { - nSigOps += txout.scriptPubKey.GetSigOpCount(false); - } - return nSigOps; -} - -unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& inputs) -{ - if (tx.IsCoinBase()) - return 0; - - unsigned int nSigOps = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); - if (prevout.scriptPubKey.IsPayToScriptHash()) - nSigOps += prevout.scriptPubKey.GetSigOpCount(tx.vin[i].scriptSig); - } - return nSigOps; -} - -int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags) -{ - int64_t nSigOps = GetLegacySigOpCount(tx) * WITNESS_SCALE_FACTOR; - - if (tx.IsCoinBase()) - return nSigOps; - - if (flags & SCRIPT_VERIFY_P2SH) { - nSigOps += GetP2SHSigOpCount(tx, inputs) * WITNESS_SCALE_FACTOR; - } - - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); - nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, &tx.vin[i].scriptWitness, flags); - } - return nSigOps; -} - - - - - -bool CheckTransaction(const CTransaction& tx, CValidationState &state, bool fCheckDuplicateInputs) -{ - // Basic checks that don't depend on any context - if (tx.vin.empty()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); - if (tx.vout.empty()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); - // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) - if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); - - // Check for negative or overflow output values - CAmount nValueOut = 0; - for (const auto& txout : tx.vout) - { - if (txout.nValue < 0) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); - if (txout.nValue > MAX_MONEY) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-toolarge"); - nValueOut += txout.nValue; - if (!MoneyRange(nValueOut)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); - } - - // Check for duplicate inputs - note that this check is slow so we skip it in CheckBlock - if (fCheckDuplicateInputs) { - std::set<COutPoint> vInOutPoints; - for (const auto& txin : tx.vin) - { - if (!vInOutPoints.insert(txin.prevout).second) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); - } - } - - if (tx.IsCoinBase()) - { - if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) - return state.DoS(100, false, REJECT_INVALID, "bad-cb-length"); - } - else - { - for (const auto& txin : tx.vin) - if (txin.prevout.IsNull()) - return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); - } - - return true; -} - void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) { int expired = pool.Expire(GetTime() - age); if (expired != 0) { LogPrint(BCLog::MEMPOOL, "Expired %i transactions from the memory pool\n", expired); } - std::vector<uint256> vNoSpendsRemaining; + std::vector<COutPoint> vNoSpendsRemaining; pool.TrimToSize(limit, &vNoSpendsRemaining); - BOOST_FOREACH(const uint256& removed, vNoSpendsRemaining) + BOOST_FOREACH(const COutPoint& removed, vNoSpendsRemaining) pcoinsTip->Uncache(removed); } @@ -537,9 +342,59 @@ static bool IsCurrentForFeeEstimation() return true; } +/* Make mempool consistent after a reorg, by re-adding or recursively erasing + * disconnected block transactions from the mempool, and also removing any + * other transactions from the mempool that are no longer valid given the new + * tip/height. + * + * Note: we assume that disconnectpool only contains transactions that are NOT + * confirmed in the current chain nor already in the mempool (otherwise, + * in-mempool descendants of such transactions would be removed). + * + * Passing fAddToMempool=false will skip trying to add the transactions back, + * and instead just erase from the mempool as needed. + */ + +void UpdateMempoolForReorg(DisconnectedBlockTransactions &disconnectpool, bool fAddToMempool) +{ + AssertLockHeld(cs_main); + std::vector<uint256> vHashUpdate; + // disconnectpool's insertion_order index sorts the entries from + // oldest to newest, but the oldest entry will be the last tx from the + // latest mined block that was disconnected. + // Iterate disconnectpool in reverse, so that we add transactions + // back to the mempool starting with the earliest transaction that had + // been previously seen in a block. + auto it = disconnectpool.queuedTx.get<insertion_order>().rbegin(); + while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { + // ignore validation errors in resurrected transactions + CValidationState stateDummy; + if (!fAddToMempool || (*it)->IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, *it, false, NULL, NULL, true)) { + // If the transaction doesn't make it in to the mempool, remove any + // transactions that depend on it (which would now be orphans). + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + } else if (mempool.exists((*it)->GetHash())) { + vHashUpdate.push_back((*it)->GetHash()); + } + ++it; + } + disconnectpool.queuedTx.clear(); + // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // no in-mempool children, which is generally not true when adding + // previously-confirmed transactions back to the mempool. + // UpdateTransactionsFromBlock finds descendants of any transactions in + // the disconnectpool that were added back and cleans up the mempool state. + mempool.UpdateTransactionsFromBlock(vHashUpdate); + + // We also need to remove any now-immature transactions + mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + // Re-limit mempool size, in case we added any transactions + LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); +} + bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool fLimitFree, bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, - bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<uint256>& vHashTxnToUncache) + bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, std::vector<COutPoint>& coins_to_uncache) { const CTransaction& tx = *ptx; const uint256 hash = tx.GetHash(); @@ -632,30 +487,30 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C view.SetBackend(viewMemPool); // do we already have it? - bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(hash); - if (view.HaveCoins(hash)) { - if (!fHadTxInCache) - vHashTxnToUncache.push_back(hash); - return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + for (size_t out = 0; out < tx.vout.size(); out++) { + COutPoint outpoint(hash, out); + bool had_coin_in_cache = pcoinsTip->HaveCoinInCache(outpoint); + if (view.HaveCoin(outpoint)) { + if (!had_coin_in_cache) { + coins_to_uncache.push_back(outpoint); + } + return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + } } // do all inputs exist? - // Note that this does not check for the presence of actual outputs (see the next check for that), - // and only helps with filling in pfMissingInputs (to determine missing vs spent). BOOST_FOREACH(const CTxIn txin, tx.vin) { - if (!pcoinsTip->HaveCoinsInCache(txin.prevout.hash)) - vHashTxnToUncache.push_back(txin.prevout.hash); - if (!view.HaveCoins(txin.prevout.hash)) { - if (pfMissingInputs) + if (!pcoinsTip->HaveCoinInCache(txin.prevout)) { + coins_to_uncache.push_back(txin.prevout); + } + if (!view.HaveCoin(txin.prevout)) { + if (pfMissingInputs) { *pfMissingInputs = true; + } return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid() } } - // are the actual inputs available? - if (!view.HaveInputs(tx)) - return state.Invalid(false, REJECT_DUPLICATE, "bad-txns-inputs-spent"); - // Bring the best block into scope view.GetBestBlock(); @@ -693,8 +548,8 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // during reorgs to ensure COINBASE_MATURITY is still met. bool fSpendsCoinbase = false; BOOST_FOREACH(const CTxIn &txin, tx.vin) { - const CCoins *coins = view.AccessCoins(txin.prevout.hash); - if (coins->IsCoinBase()) { + const Coin &coin = view.AccessCoin(txin.prevout); + if (coin.IsCoinBase()) { fSpendsCoinbase = true; break; } @@ -958,10 +813,10 @@ bool AcceptToMemoryPoolWithTime(CTxMemPool& pool, CValidationState &state, const bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) { - std::vector<uint256> vHashTxToUncache; - bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + std::vector<COutPoint> coins_to_uncache; + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, nAcceptTime, plTxnReplaced, fOverrideMempoolLimit, nAbsurdFee, coins_to_uncache); if (!res) { - BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) + BOOST_FOREACH(const COutPoint& hashTx, coins_to_uncache) pcoinsTip->Uncache(hashTx); } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits @@ -1013,15 +868,8 @@ bool GetTransaction(const uint256 &hash, CTransactionRef &txOut, const Consensus } if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - int nHeight = -1; - { - const CCoinsViewCache& view = *pcoinsTip; - const CCoins* coins = view.AccessCoins(hash); - if (coins) - nHeight = coins->nHeight; - } - if (nHeight > 0) - pindexSlow = chainActive[nHeight]; + const Coin& coin = AccessByTxid(*pcoinsTip, hash); + if (!coin.IsSpent()) pindexSlow = chainActive[coin.nHeight]; } if (pindexSlow) { @@ -1139,6 +987,7 @@ bool IsInitialBlockDownload() return true; if (chainActive.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) return true; + LogPrintf("Leaving InitialBlockDownload (latching to false)\n"); latchToFalse.store(true, std::memory_order_relaxed); return false; } @@ -1268,24 +1117,12 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund if (!tx.IsCoinBase()) { txundo.vprevout.reserve(tx.vin.size()); BOOST_FOREACH(const CTxIn &txin, tx.vin) { - CCoinsModifier coins = inputs.ModifyCoins(txin.prevout.hash); - unsigned nPos = txin.prevout.n; - - if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull()) - assert(false); - // mark an outpoint spent, and construct undo information - txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos])); - coins->Spend(nPos); - if (coins->vout.size() == 0) { - CTxInUndo& undo = txundo.vprevout.back(); - undo.nHeight = coins->nHeight; - undo.fCoinBase = coins->fCoinBase; - undo.nVersion = coins->nVersion; - } + txundo.vprevout.emplace_back(); + inputs.SpendCoin(txin.prevout, &txundo.vprevout.back()); } } // add outputs - inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight); + AddCoins(inputs, tx, nHeight); } void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) @@ -1297,10 +1134,7 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) bool CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; const CScriptWitness *witness = &ptxTo->vin[nIn].scriptWitness; - if (!VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore, *txdata), &error)) { - return false; - } - return true; + return VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore, *txdata), &error); } int GetSpendHeight(const CCoinsViewCache& inputs) @@ -1310,52 +1144,6 @@ int GetSpendHeight(const CCoinsViewCache& inputs) return pindexPrev->nHeight + 1; } -namespace Consensus { -bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight) -{ - // This doesn't trigger the DoS code on purpose; if it did, it would make it easier - // for an attacker to attempt to split the network. - if (!inputs.HaveInputs(tx)) - return state.Invalid(false, 0, "", "Inputs unavailable"); - - CAmount nValueIn = 0; - CAmount nFees = 0; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins *coins = inputs.AccessCoins(prevout.hash); - assert(coins); - - // If prev is coinbase, check that it's matured - if (coins->IsCoinBase()) { - if (nSpendHeight - coins->nHeight < COINBASE_MATURITY) - return state.Invalid(false, - REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", - strprintf("tried to spend coinbase at depth %d", nSpendHeight - coins->nHeight)); - } - - // Check for negative or overflow input values - nValueIn += coins->vout[prevout.n].nValue; - if (!MoneyRange(coins->vout[prevout.n].nValue) || !MoneyRange(nValueIn)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); - - } - - if (nValueIn < tx.GetValueOut()) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, - strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(tx.GetValueOut()))); - - // Tally transaction fees - CAmount nTxFee = nValueIn - tx.GetValueOut(); - if (nTxFee < 0) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-negative"); - nFees += nTxFee; - if (!MoneyRange(nFees)) - return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); - return true; -} -}// namespace Consensus - bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks) { if (!tx.IsCoinBase()) @@ -1378,11 +1166,19 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi if (fScriptChecks) { for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; - const CCoins* coins = inputs.AccessCoins(prevout.hash); - assert(coins); + const Coin& coin = inputs.AccessCoin(prevout); + assert(!coin.IsSpent()); + + // We very carefully only pass in things to CScriptCheck which + // are clearly committed to by tx' witness hash. This provides + // a sanity check that our caching is not introducing consensus + // failures through additional data in, eg, the coins being + // spent being checked as a part of CScriptCheck. + const CScript& scriptPubKey = coin.out.scriptPubKey; + const CAmount amount = coin.out.nValue; // Verify signature - CScriptCheck check(*coins, tx, i, flags, cacheStore, &txdata); + CScriptCheck check(scriptPubKey, amount, tx, i, flags, cacheStore, &txdata); if (pvChecks) { pvChecks->push_back(CScriptCheck()); check.swap(pvChecks->back()); @@ -1394,7 +1190,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // arguments; if so, don't trigger DoS protection to // avoid splitting the network between upgraded and // non-upgraded nodes. - CScriptCheck check2(*coins, tx, i, + CScriptCheck check2(scriptPubKey, amount, tx, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheStore, &txdata); if (check2()) return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); @@ -1453,8 +1249,10 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin // Read block uint256 hashChecksum; + CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data try { - filein >> blockundo; + verifier << hashBlock; + verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { @@ -1462,10 +1260,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uin } // Verify checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - if (hashChecksum != hasher.GetHash()) + if (hashChecksum != verifier.GetHash()) return error("%s: Checksum mismatch", __func__); return true; @@ -1491,57 +1286,66 @@ bool AbortNode(CValidationState& state, const std::string& strMessage, const std } // anon namespace +enum DisconnectResult +{ + DISCONNECT_OK, // All good. + DISCONNECT_UNCLEAN, // Rolled back, but UTXO set was inconsistent with block. + DISCONNECT_FAILED // Something else went wrong. +}; + /** - * Apply the undo operation of a CTxInUndo to the given chain state. - * @param undo The undo object. + * Restore the UTXO in a Coin at a given COutPoint + * @param undo The Coin to be restored. * @param view The coins view to which to apply the changes. * @param out The out point that corresponds to the tx input. - * @return True on success. + * @return A DisconnectResult as an int */ -bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) +int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) { bool fClean = true; - CCoinsModifier coins = view.ModifyCoins(out.hash); - if (undo.nHeight != 0) { - // undo data contains height: this is the last output of the prevout tx being spent - if (!coins->IsPruned()) - fClean = fClean && error("%s: undo data overwriting existing transaction", __func__); - coins->Clear(); - coins->fCoinBase = undo.fCoinBase; - coins->nHeight = undo.nHeight; - coins->nVersion = undo.nVersion; - } else { - if (coins->IsPruned()) - fClean = fClean && error("%s: undo data adding output to missing transaction", __func__); + if (view.HaveCoin(out)) fClean = false; // overwriting transaction output + + if (undo.nHeight == 0) { + // Missing undo metadata (height and coinbase). Older versions included this + // information only in undo records for the last spend of a transactions' + // outputs. This implies that it must be present for some other output of the same tx. + const Coin& alternate = AccessByTxid(view, out.hash); + if (!alternate.IsSpent()) { + undo.nHeight = alternate.nHeight; + undo.fCoinBase = alternate.fCoinBase; + } else { + return DISCONNECT_FAILED; // adding output for transaction without known metadata + } } - if (coins->IsAvailable(out.n)) - fClean = fClean && error("%s: undo data overwriting existing output", __func__); - if (coins->vout.size() < out.n+1) - coins->vout.resize(out.n+1); - coins->vout[out.n] = undo.txout; + view.AddCoin(out, std::move(undo), undo.fCoinBase); - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) +/** Undo the effects of this block (with given index) on the UTXO set represented by coins. + * When UNCLEAN or FAILED is returned, view is left in an indeterminate state. */ +static DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) { assert(pindex->GetBlockHash() == view.GetBestBlock()); - if (pfClean) - *pfClean = false; - bool fClean = true; CBlockUndo blockUndo; CDiskBlockPos pos = pindex->GetUndoPos(); - if (pos.IsNull()) - return error("DisconnectBlock(): no undo data available"); - if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) - return error("DisconnectBlock(): failure reading undo data"); + if (pos.IsNull()) { + error("DisconnectBlock(): no undo data available"); + return DISCONNECT_FAILED; + } + if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) { + error("DisconnectBlock(): failure reading undo data"); + return DISCONNECT_FAILED; + } - if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) - return error("DisconnectBlock(): block and undo data inconsistent"); + if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) { + error("DisconnectBlock(): block and undo data inconsistent"); + return DISCONNECT_FAILED; + } // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -1550,46 +1354,38 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI // Check that all outputs are available and match the outputs in the block itself // exactly. - { - CCoinsModifier outs = view.ModifyCoins(hash); - outs->ClearUnspendable(); - - CCoins outsBlock(tx, pindex->nHeight); - // The CCoins serialization does not serialize negative numbers. - // No network rules currently depend on the version here, so an inconsistency is harmless - // but it must be corrected before txout nversion ever influences a network rule. - if (outsBlock.nVersion < 0) - outs->nVersion = outsBlock.nVersion; - if (*outs != outsBlock) - fClean = fClean && error("DisconnectBlock(): added transaction mismatch? database corrupted"); - - // remove outputs - outs->Clear(); + for (size_t o = 0; o < tx.vout.size(); o++) { + if (!tx.vout[o].scriptPubKey.IsUnspendable()) { + COutPoint out(hash, o); + Coin coin; + view.SpendCoin(out, &coin); + if (tx.vout[o] != coin.out) { + fClean = false; // transaction output mismatch + } + } } // restore inputs if (i > 0) { // not coinbases - const CTxUndo &txundo = blockUndo.vtxundo[i-1]; - if (txundo.vprevout.size() != tx.vin.size()) - return error("DisconnectBlock(): transaction and undo data inconsistent"); + CTxUndo &txundo = blockUndo.vtxundo[i-1]; + if (txundo.vprevout.size() != tx.vin.size()) { + error("DisconnectBlock(): transaction and undo data inconsistent"); + return DISCONNECT_FAILED; + } for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; - const CTxInUndo &undo = txundo.vprevout[j]; - if (!ApplyTxInUndo(undo, view, out)) - fClean = false; + int res = ApplyTxInUndo(std::move(txundo.vprevout[j]), view, out); + if (res == DISCONNECT_FAILED) return DISCONNECT_FAILED; + fClean = fClean && res != DISCONNECT_UNCLEAN; } + // At this point, all of txundo.vprevout should have been moved out. } } // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); - if (pfClean) { - *pfClean = fClean; - return true; - } - - return fClean; + return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } void static FlushBlockFile(bool fFinalize = false) @@ -1677,8 +1473,11 @@ static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) +/** Apply the effects of this block (with given index) on the UTXO set represented by coins. + * Validity checks that depend on the UTXO set are also done; ConnectBlock() + * can fail if those validity checks fail (among other reasons). */ +static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) { AssertLockHeld(cs_main); assert(pindex); @@ -1760,10 +1559,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (fEnforceBIP30) { for (const auto& tx : block.vtx) { - const CCoins* coins = view.AccessCoins(tx->GetHash()); - if (coins && !coins->IsPruned()) - return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), - REJECT_INVALID, "bad-txns-BIP30"); + for (size_t o = 0; o < tx->vout.size(); o++) { + if (view.HaveCoin(COutPoint(tx->GetHash(), o))) { + return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), + REJECT_INVALID, "bad-txns-BIP30"); + } + } } } @@ -1830,7 +1631,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin // be in ConnectBlock because they require the UTXO set prevheights.resize(tx.vin.size()); for (size_t j = 0; j < tx.vin.size(); j++) { - prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; + prevheights[j] = view.AccessCoin(tx.vin[j].prevout).nHeight; } if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { @@ -1917,12 +1718,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime5 - nTime4), nTimeIndex * 0.000001); - // Watch for changes to the previous coinbase transaction. - static uint256 hashPrevBestCoinBase; - GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); - hashPrevBestCoinBase = block.vtx[0]->GetHash(); - - int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; LogPrint(BCLog::BENCH, " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); @@ -1974,9 +1769,8 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max<int64_t>(nMempoolSizeMax - nMempoolUsage, 0); - // The cache is large and we're within 10% and 200 MiB or 50% and 50MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::min(std::max(nTotalSpace / 2, nTotalSpace - MIN_BLOCK_COINSDB_USAGE * 1024 * 1024), - std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024)); + // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. @@ -2017,12 +1811,12 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush) { - // Typical CCoins structures on disk are around 128 bytes in size. + // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) + if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) return state.Error("out of disk space"); // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) @@ -2051,6 +1845,16 @@ void PruneAndFlush() { FlushStateToDisk(state, FLUSH_STATE_NONE); } +static void DoWarning(const std::string& strWarning) +{ + static bool fWarned = false; + SetMiscWarning(strWarning); + if (!fWarned) { + AlertNotify(strWarning); + fWarned = true; + } +} + /** Update chainActive and related internal data structures. */ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.SetTip(pindexNew); @@ -2060,7 +1864,6 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { cvBlockChange.notify_all(); - static bool fWarned = false; std::vector<std::string> warningMessages; if (!IsInitialBlockDownload()) { @@ -2070,15 +1873,11 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { WarningBitsConditionChecker checker(bit); ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); if (state == THRESHOLD_ACTIVE || state == THRESHOLD_LOCKED_IN) { + const std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); if (state == THRESHOLD_ACTIVE) { - std::string strWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); - SetMiscWarning(strWarning); - if (!fWarned) { - AlertNotify(strWarning); - fWarned = true; - } + DoWarning(strWarning); } else { - warningMessages.push_back(strprintf("unknown new rules are about to activate (versionbit %i)", bit)); + warningMessages.push_back(strWarning); } } } @@ -2091,19 +1890,15 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { pindex = pindex->pprev; } if (nUpgraded > 0) - warningMessages.push_back(strprintf("%d of last 100 blocks have unexpected version", nUpgraded)); + warningMessages.push_back(strprintf(_("%d of last 100 blocks have unexpected version"), nUpgraded)); if (nUpgraded > 100/2) { std::string strWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); // notify GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - SetMiscWarning(strWarning); - if (!fWarned) { - AlertNotify(strWarning); - fWarned = true; - } + DoWarning(strWarning); } } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__, + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), @@ -2114,8 +1909,17 @@ void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { } -/** Disconnect chainActive's tip. You probably want to call mempool.removeForReorg and manually re-limit mempool size after this, with cs_main held. */ -bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, bool fBare = false) +/** Disconnect chainActive's tip. + * After calling, the mempool will be in an inconsistent state, with + * transactions from disconnected blocks being added to disconnectpool. You + * should make the mempool consistent again by calling UpdateMempoolForReorg. + * with cs_main held. + * + * If disconnectpool is NULL, then no disconnected transactions are added to + * disconnectpool (note that the caller is responsible for mempool consistency + * in any case). + */ +bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions *disconnectpool) { CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); @@ -2128,7 +1932,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara int64_t nStart = GetTimeMicros(); { CCoinsViewCache view(pcoinsTip); - if (!DisconnectBlock(block, state, pindexDelete, view)) + if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); bool flushed = view.Flush(); assert(flushed); @@ -2138,25 +1942,17 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; - if (!fBare) { - // Resurrect mempool transactions from the disconnected block. - std::vector<uint256> vHashUpdate; - for (const auto& it : block.vtx) { - const CTransaction& tx = *it; - // ignore validation errors in resurrected transactions - CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, it, false, NULL, NULL, true)) { - mempool.removeRecursive(tx, MemPoolRemovalReason::REORG); - } else if (mempool.exists(tx.GetHash())) { - vHashUpdate.push_back(tx.GetHash()); - } + if (disconnectpool) { + // Save transactions to re-add to mempool at end of reorg + for (auto it = block.vtx.rbegin(); it != block.vtx.rend(); ++it) { + disconnectpool->addTransaction(*it); + } + while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE * 1000) { + // Drop the earliest entry, and remove its children from the mempool. + auto it = disconnectpool->queuedTx.get<insertion_order>().begin(); + mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + disconnectpool->removeEntry(it); } - // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have - // no in-mempool children, which is generally not true when adding - // previously-confirmed transactions back to the mempool. - // UpdateTransactionsFromBlock finds descendants of any transactions in this - // block that were added back and cleans up the mempool state. - mempool.UpdateTransactionsFromBlock(vHashUpdate); } // Update chainActive and related variables. @@ -2244,7 +2040,7 @@ public: * * The block is added to connectTrace if connection succeeds. */ -bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace) +bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) { assert(pindexNew->pprev == chainActive.Tip()); // Read block from disk. @@ -2286,6 +2082,7 @@ bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool.; mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); + disconnectpool.removeForBlock(blockConnecting.vtx); // Update chainActive & related variables. UpdateTip(pindexNew, chainparams); @@ -2379,9 +2176,14 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // Disconnect active blocks which are no longer in the best chain. bool fBlocksDisconnected = false; + DisconnectedBlockTransactions disconnectpool; while (chainActive.Tip() && chainActive.Tip() != pindexFork) { - if (!DisconnectTip(state, chainparams)) + if (!DisconnectTip(state, chainparams, &disconnectpool)) { + // This is likely a fatal error, but keep the mempool consistent, + // just in case. Only remove from the mempool in this case. + UpdateMempoolForReorg(disconnectpool, false); return false; + } fBlocksDisconnected = true; } @@ -2404,7 +2206,7 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c // Connect new blocks. BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { - if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace)) { + if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. if (!state.CorruptionPossible()) @@ -2415,6 +2217,9 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c break; } else { // A system error occurred (disk space, database error, ...). + // Make the mempool consistent with the current tip, just in case + // any observers try to use it before shutdown. + UpdateMempoolForReorg(disconnectpool, false); return false; } } else { @@ -2429,8 +2234,9 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c } if (fBlocksDisconnected) { - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); - LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + // If any blocks were disconnected, disconnectpool may be non empty. Add + // any disconnected transactions back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); } mempool.check(pcoinsTip); @@ -2534,6 +2340,9 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, return false; } + int nStopAtHeight = GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); + if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) StartShutdown(); + return true; } @@ -2576,6 +2385,7 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C setDirtyBlockIndex.insert(pindex); setBlockIndexCandidates.erase(pindex); + DisconnectedBlockTransactions disconnectpool; while (chainActive.Contains(pindex)) { CBlockIndex *pindexWalk = chainActive.Tip(); pindexWalk->nStatus |= BLOCK_FAILED_CHILD; @@ -2583,13 +2393,17 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C setBlockIndexCandidates.erase(pindexWalk); // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. - if (!DisconnectTip(state, chainparams)) { - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + if (!DisconnectTip(state, chainparams, &disconnectpool)) { + // It's probably hopeless to try to make the mempool consistent + // here if DisconnectTip failed, but we can try. + UpdateMempoolForReorg(disconnectpool, false); return false; } } - LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + // DisconnectTip will add transactions to disconnectpool; try to add these + // back to the mempool. + UpdateMempoolForReorg(disconnectpool, true); // The resulting new best tip may not be in setBlockIndexCandidates anymore, so // add it again. @@ -2602,7 +2416,6 @@ bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, C } InvalidChainFound(pindex); - mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); return true; } @@ -2676,7 +2489,7 @@ CBlockIndex* AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos) +static bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2684,7 +2497,7 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; - if (IsWitnessEnabled(pindexNew->pprev, Params().GetConsensus())) { + if (IsWitnessEnabled(pindexNew->pprev, consensusParams)) { pindexNew->nStatus |= BLOCK_OPT_WITNESS; } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); @@ -3050,8 +2863,8 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Co // No witness data is allowed in blocks that don't commit to witness data, as this would otherwise leave room for spam if (!fHaveWitness) { - for (size_t i = 0; i < block.vtx.size(); i++) { - if (block.vtx[i]->HasWitness()) { + for (const auto& tx : block.vtx) { + if (tx->HasWitness()) { return state.DoS(100, false, REJECT_INVALID, "unexpected-witness", true, strprintf("%s : unexpected witness data found", __func__)); } } @@ -3206,7 +3019,7 @@ static bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidation if (dbp == NULL) if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) AbortNode(state, "Failed to write block"); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("AcceptBlock(): ReceivedBlockTransactions failed"); } catch (const std::runtime_error& e) { return AbortNode(state, std::string("System error: ") + e.what()); @@ -3653,15 +3466,17 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { - bool fClean = true; - if (!DisconnectBlock(block, state, pindex, coins, &fClean)) + DisconnectResult res = DisconnectBlock(block, pindex, coins); + if (res == DISCONNECT_FAILED) { return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + } pindexState = pindex->pprev; - if (!fClean) { + if (res == DISCONNECT_UNCLEAN) { nGoodTransactions = 0; pindexFailure = pindex; - } else + } else { nGoodTransactions += block.vtx.size(); + } } if (ShutdownRequested()) return true; @@ -3714,7 +3529,7 @@ bool RewindBlockIndex(const CChainParams& params) // of the blockchain). break; } - if (!DisconnectTip(state, params, true)) { + if (!DisconnectTip(state, params, NULL)) { return error("RewindBlockIndex: unable to disconnect block at height %i", pindex->nHeight); } // Occasionally flush state to disk. @@ -3837,7 +3652,7 @@ bool InitBlockIndex(const CChainParams& chainparams) if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) return error("LoadBlockIndex(): writing genesis block to disk failed"); CBlockIndex *pindex = AddToBlockIndex(block); - if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) + if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) return error("LoadBlockIndex(): genesis block not accepted"); // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); @@ -4166,6 +3981,12 @@ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::D return VersionBitsState(chainActive.Tip(), params, pos, versionbitscache); } +BIP9Stats VersionBitsTipStatistics(const Consensus::Params& params, Consensus::DeploymentPos pos) +{ + LOCK(cs_main); + return VersionBitsStatistics(chainActive.Tip(), params, pos); +} + int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::DeploymentPos pos) { LOCK(cs_main); diff --git a/src/validation.h b/src/validation.h index 4aa10cbb0b..096fd0a9ee 100644 --- a/src/validation.h +++ b/src/validation.h @@ -11,10 +11,10 @@ #endif #include "amount.h" -#include "chain.h" #include "coins.h" #include "fs.h" #include "protocol.h" // For CMessageHeader::MessageStartChars +#include "policy/feerate.h" #include "script/script_error.h" #include "sync.h" #include "versionbits.h" @@ -39,6 +39,7 @@ class CChainParams; class CInv; class CConnman; class CScriptCheck; +class CBlockPolicyEstimator; class CTxMemPool; class CValidationInterface; class CValidationState; @@ -69,6 +70,8 @@ static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25; static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; +/** Maximum kilobytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ @@ -104,7 +107,7 @@ static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; /** Maximum length of reject messages. */ static const unsigned int MAX_REJECT_MESSAGE_LENGTH = 111; /** Average delay between local address broadcasts in seconds. */ -static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 24 * 60; +static const unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 60 * 60; /** Average delay between peer address broadcasts in seconds. */ static const unsigned int AVG_ADDRESS_BROADCAST_INTERVAL = 30; /** Average delay between trickled inventory transmissions in seconds. @@ -131,7 +134,8 @@ static const bool DEFAULT_PERMIT_BAREMULTISIG = true; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; - +/** Default for -persistmempool */ +static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -mempoolreplacement */ static const bool DEFAULT_ENABLE_REPLACEMENT = true; /** Default for using fee filter */ @@ -145,6 +149,9 @@ static const int MAX_UNCONNECTING_HEADERS = 10; static const bool DEFAULT_PEERBLOOMFILTERS = true; +/** Default for -stopatheight */ +static const int DEFAULT_STOPATHEIGHT = 0; + struct BlockHasher { size_t operator()(const uint256& hash) const { return hash.GetCheapHash(); } @@ -152,6 +159,7 @@ struct BlockHasher extern CScript COINBASE_FLAGS; extern CCriticalSection cs_main; +extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; typedef boost::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern BlockMap mapBlockIndex; @@ -333,33 +341,12 @@ std::string FormatStateMessage(const CValidationState &state); /** Get the BIP9 state for a given deployment at the current tip. */ ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos); +/** Get the numerical statistics for the BIP9 state for a given deployment at the current tip. */ +BIP9Stats VersionBitsTipStatistics(const Consensus::Params& params, Consensus::DeploymentPos pos); + /** Get the block height at which the BIP9 deployment switched into the state for the block building on the current tip. */ int VersionBitsTipStateSinceHeight(const Consensus::Params& params, Consensus::DeploymentPos pos); -/** - * Count ECDSA signature operations the old-fashioned (pre-0.6) way - * @return number of sigops this transaction's outputs will produce when spent - * @see CTransaction::FetchInputs - */ -unsigned int GetLegacySigOpCount(const CTransaction& tx); - -/** - * Count ECDSA signature operations in pay-to-script-hash inputs. - * - * @param[in] mapInputs Map of previous transactions that have outputs we're spending - * @return maximum number of sigops required to validate this transaction's inputs - * @see CTransaction::FetchInputs - */ -unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& mapInputs); - -/** - * Compute total signature operation cost of a transaction. - * @param[in] tx Transaction for which we are computing the cost - * @param[in] inputs Map of previous transactions that have outputs we're spending - * @param[out] flags Script verification flags - * @return Total signature operation cost of tx - */ -int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags); /** * Check whether all inputs of this transaction are valid (no double spends, scripts & sigs, amounts) @@ -374,26 +361,6 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight); /** Transaction validation functions */ -/** Context-independent validity checks */ -bool CheckTransaction(const CTransaction& tx, CValidationState& state, bool fCheckDuplicateInputs=true); - -namespace Consensus { - -/** - * Check whether all inputs of this transaction are valid (no double spends and amounts) - * This does not modify the UTXO set. This does not check scripts and sigs. - * Preconditions: tx.IsCoinBase() is false. - */ -bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight); - -} // namespace Consensus - -/** - * Check if transaction is final and can be included in a block with the - * specified height and time. Consensus critical. - */ -bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime); - /** * Check if transaction will be final in the next block to be created. * @@ -409,12 +376,6 @@ bool CheckFinalTx(const CTransaction &tx, int flags = -1); bool TestLockPointValidity(const LockPoints* lp); /** - * Check if transaction is final per BIP 68 sequence numbers and can be included in a block. - * Consensus critical. Takes as input a list of heights at which tx's inputs (in order) confirmed. - */ -bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block); - -/** * Check if transaction will be BIP 68 final in the next block to be created. * * Simulates calling SequenceLocks() with data from the tip of the current active chain. @@ -445,8 +406,8 @@ private: public: CScriptCheck(): amount(0), ptxTo(0), nIn(0), nFlags(0), cacheStore(false), error(SCRIPT_ERR_UNKNOWN_ERROR) {} - CScriptCheck(const CCoins& txFromIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : - scriptPubKey(txFromIn.vout[txToIn.vin[nInIn].prevout.n].scriptPubKey), amount(txFromIn.vout[txToIn.vin[nInIn].prevout.n].nValue), + CScriptCheck(const CScript& scriptPubKeyIn, const CAmount amountIn, const CTransaction& txToIn, unsigned int nInIn, unsigned int nFlagsIn, bool cacheIn, PrecomputedTransactionData* txdataIn) : + scriptPubKey(scriptPubKeyIn), amount(amountIn), ptxTo(&txToIn), nIn(nInIn), nFlags(nFlagsIn), cacheStore(cacheIn), error(SCRIPT_ERR_UNKNOWN_ERROR), txdata(txdataIn) { } bool operator()(); @@ -483,18 +444,6 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime); bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); -/** Apply the effects of this block (with given index) on the UTXO set represented by coins. - * Validity checks that depend on the UTXO set are also done; ConnectBlock() - * can fail if those validity checks fail (among other reasons). */ -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& coins, - const CChainParams& chainparams, bool fJustCheck = false); - -/** Undo the effects of this block (with given index) on the UTXO set represented by coins. - * In case pfClean is provided, operation will try to be tolerant about errors, and *pfClean - * will be true if no problems were found. Otherwise, the return value will be false in case - * of problems. Note that in any case, coins may be modified. */ -bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& coins, bool* pfClean = NULL); - /** Check a block is completely valid from start to finish (only works on top of our current best block, with cs_main held) */ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 0f699328c7..46d7c9b329 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -17,7 +17,6 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.TransactionAddedToMempool.connect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); g_signals.BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); g_signals.BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); - g_signals.UpdatedTransaction.connect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); @@ -32,7 +31,6 @@ void UnregisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.Broadcast.disconnect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); g_signals.Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); - g_signals.UpdatedTransaction.disconnect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); g_signals.TransactionAddedToMempool.disconnect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); g_signals.BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); g_signals.BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); @@ -46,7 +44,6 @@ void UnregisterAllValidationInterfaces() { g_signals.Broadcast.disconnect_all_slots(); g_signals.Inventory.disconnect_all_slots(); g_signals.SetBestChain.disconnect_all_slots(); - g_signals.UpdatedTransaction.disconnect_all_slots(); g_signals.TransactionAddedToMempool.disconnect_all_slots(); g_signals.BlockConnected.disconnect_all_slots(); g_signals.BlockDisconnected.disconnect_all_slots(); diff --git a/src/validationinterface.h b/src/validationinterface.h index 083c136f2c..460aecf243 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -37,7 +37,6 @@ protected: virtual void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex, const std::vector<CTransactionRef> &txnConflicted) {} virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} virtual void SetBestChain(const CBlockLocator &locator) {} - virtual void UpdatedTransaction(const uint256 &hash) {} virtual void Inventory(const uint256 &hash) {} virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} virtual void BlockChecked(const CBlock&, const CValidationState&) {} @@ -60,8 +59,6 @@ struct CMainSignals { boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef> &)> BlockConnected; /** Notifies listeners of a block being disconnected */ boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; - /** Notifies listeners of an updated transaction without new data (for now: a coinbase potentially becoming visible). */ - boost::signals2::signal<void (const uint256 &)> UpdatedTransaction; /** Notifies listeners of a new active block chain. */ boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; /** Notifies listeners about an inventory item being seen on the network. */ diff --git a/src/versionbits.cpp b/src/versionbits.cpp index 8a7cce7485..80786233f5 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "versionbits.h" - #include "consensus/params.h" const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { @@ -105,6 +104,36 @@ ThresholdState AbstractThresholdConditionChecker::GetStateFor(const CBlockIndex* return state; } +// return the numerical statistics of blocks signalling the specified BIP9 condition in this current period +BIP9Stats AbstractThresholdConditionChecker::GetStateStatisticsFor(const CBlockIndex* pindex, const Consensus::Params& params) const +{ + BIP9Stats stats; + + stats.period = Period(params); + stats.threshold = Threshold(params); + + if (pindex == NULL) + return stats; + + // Find beginning of period + const CBlockIndex* pindexEndOfPrevPeriod = pindex->GetAncestor(pindex->nHeight - ((pindex->nHeight + 1) % stats.period)); + stats.elapsed = pindex->nHeight - pindexEndOfPrevPeriod->nHeight; + + // Count from current block to beginning of period + int count = 0; + const CBlockIndex* currentIndex = pindex; + while (pindexEndOfPrevPeriod->nHeight != currentIndex->nHeight){ + if (Condition(currentIndex, params)) + count++; + currentIndex = currentIndex->pprev; + } + + stats.count = count; + stats.possible = (stats.period - stats.threshold ) >= (stats.elapsed - count); + + return stats; +} + int AbstractThresholdConditionChecker::GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const { const ThresholdState initialState = GetStateFor(pindexPrev, params, cache); @@ -167,6 +196,11 @@ ThresholdState VersionBitsState(const CBlockIndex* pindexPrev, const Consensus:: return VersionBitsConditionChecker(pos).GetStateFor(pindexPrev, params, cache.caches[pos]); } +BIP9Stats VersionBitsStatistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos) +{ + return VersionBitsConditionChecker(pos).GetStateStatisticsFor(pindexPrev, params); +} + int VersionBitsStateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache) { return VersionBitsConditionChecker(pos).GetStateSinceHeightFor(pindexPrev, params, cache.caches[pos]); diff --git a/src/versionbits.h b/src/versionbits.h index 7a929266aa..f1d31ea0af 100644 --- a/src/versionbits.h +++ b/src/versionbits.h @@ -37,6 +37,14 @@ struct BIP9DeploymentInfo { bool gbt_force; }; +struct BIP9Stats { + int period; + int threshold; + int elapsed; + int count; + bool possible; +}; + extern const struct BIP9DeploymentInfo VersionBitsDeploymentInfo[]; /** @@ -51,6 +59,7 @@ protected: virtual int Threshold(const Consensus::Params& params) const =0; public: + BIP9Stats GetStateStatisticsFor(const CBlockIndex* pindex, const Consensus::Params& params) const; // Note that the functions below take a pindexPrev as input: they compute information for block B based on its parent. ThresholdState GetStateFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const; int GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const; @@ -64,6 +73,7 @@ struct VersionBitsCache }; ThresholdState VersionBitsState(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache); +BIP9Stats VersionBitsStatistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos); int VersionBitsStateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache); uint32_t VersionBitsMask(const Consensus::Params& params, Consensus::DeploymentPos pos); diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 4e93e929be..cb4719ae90 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_COINCONTROL_H #define BITCOIN_WALLET_COINCONTROL_H +#include "policy/feerate.h" #include "primitives/transaction.h" #include "wallet/wallet.h" @@ -17,8 +18,6 @@ public: bool fAllowOtherInputs; //! Includes watch only addresses which match the ISMINE_WATCH_SOLVABLE criteria bool fAllowWatchOnly; - //! Minimum absolute fee (not per kilobyte) - CAmount nMinimumTotalFee; //! Override estimated feerate bool fOverrideFeeRate; //! Feerate to use if overrideFeeRate is true @@ -39,7 +38,6 @@ public: fAllowOtherInputs = false; fAllowWatchOnly = false; setSelected.clear(); - nMinimumTotalFee = 0; nFeeRate = CFeeRate(0); fOverrideFeeRate = false; nConfirmTarget = 0; diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index f47fc92b57..25f6bdd9d9 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -359,13 +359,16 @@ void CDBEnv::CheckpointLSN(const std::string& strFile) } -CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL) +CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL) { int ret; fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); fFlushOnClose = fFlushOnCloseIn; - if (strFilename.empty()) + env = dbw.env; + if (dbw.IsDummy()) { return; + } + const std::string &strFilename = dbw.strFile; bool fCreate = strchr(pszMode, 'c') != NULL; unsigned int nFlags = DB_THREAD; @@ -373,17 +376,17 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose nFlags |= DB_CREATE; { - LOCK(bitdb.cs_db); - if (!bitdb.Open(GetDataDir())) + LOCK(env->cs_db); + if (!env->Open(GetDataDir())) throw std::runtime_error("CDB: Failed to open database environment."); strFile = strFilename; - ++bitdb.mapFileUseCount[strFile]; - pdb = bitdb.mapDb[strFile]; + ++env->mapFileUseCount[strFile]; + pdb = env->mapDb[strFile]; if (pdb == NULL) { - pdb = new Db(bitdb.dbenv, 0); + pdb = new Db(env->dbenv, 0); - bool fMockDb = bitdb.IsMock(); + bool fMockDb = env->IsMock(); if (fMockDb) { DbMpoolFile* mpf = pdb->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); @@ -401,7 +404,7 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose if (ret != 0) { delete pdb; pdb = NULL; - --bitdb.mapFileUseCount[strFile]; + --env->mapFileUseCount[strFile]; strFile = ""; throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename)); } @@ -413,7 +416,7 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose fReadOnly = fTmp; } - bitdb.mapDb[strFile] = pdb; + env->mapDb[strFile] = pdb; } } } @@ -428,7 +431,7 @@ void CDB::Flush() if (fReadOnly) nMinutes = 1; - bitdb.dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + env->dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } void CDB::Close() @@ -444,8 +447,8 @@ void CDB::Close() Flush(); { - LOCK(bitdb.cs_db); - --bitdb.mapFileUseCount[strFile]; + LOCK(env->cs_db); + --env->mapFileUseCount[strFile]; } } @@ -463,32 +466,28 @@ void CDBEnv::CloseDb(const std::string& strFile) } } -bool CDBEnv::RemoveDb(const std::string& strFile) -{ - this->CloseDb(strFile); - - LOCK(cs_db); - int rc = dbenv->dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT); - return (rc == 0); -} - -bool CDB::Rewrite(const std::string& strFile, const char* pszSkip) +bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip) { + if (dbw.IsDummy()) { + return true; + } + CDBEnv *env = dbw.env; + const std::string& strFile = dbw.strFile; while (true) { { - LOCK(bitdb.cs_db); - if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0) { + LOCK(env->cs_db); + if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { // Flush log data to the dat file - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); - bitdb.mapFileUseCount.erase(strFile); + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(strFile); bool fSuccess = true; LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile); std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} - CDB db(strFile.c_str(), "r"); - Db* pdbCopy = new Db(bitdb.dbenv, 0); + CDB db(dbw, "r"); + Db* pdbCopy = new Db(env->dbenv, 0); int ret = pdbCopy->open(NULL, // Txn pointer strFileRes.c_str(), // Filename @@ -531,17 +530,17 @@ bool CDB::Rewrite(const std::string& strFile, const char* pszSkip) } if (fSuccess) { db.Close(); - bitdb.CloseDb(strFile); + env->CloseDb(strFile); if (pdbCopy->close(0)) fSuccess = false; delete pdbCopy; } } if (fSuccess) { - Db dbA(bitdb.dbenv, 0); + Db dbA(env->dbenv, 0); if (dbA.remove(strFile.c_str(), NULL, 0)) fSuccess = false; - Db dbB(bitdb.dbenv, 0); + Db dbB(env->dbenv, 0); if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0)) fSuccess = false; } @@ -596,16 +595,21 @@ void CDBEnv::Flush(bool fShutdown) } } -bool CDB::PeriodicFlush(std::string strFile) +bool CDB::PeriodicFlush(CWalletDBWrapper& dbw) { + if (dbw.IsDummy()) { + return true; + } bool ret = false; + CDBEnv *env = dbw.env; + const std::string& strFile = dbw.strFile; TRY_LOCK(bitdb.cs_db,lockDb); if (lockDb) { // Don't do this if any databases are in use int nRefCount = 0; - std::map<std::string, int>::iterator mit = bitdb.mapFileUseCount.begin(); - while (mit != bitdb.mapFileUseCount.end()) + std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin(); + while (mit != env->mapFileUseCount.end()) { nRefCount += (*mit).second; mit++; @@ -614,17 +618,17 @@ bool CDB::PeriodicFlush(std::string strFile) if (nRefCount == 0) { boost::this_thread::interruption_point(); - std::map<std::string, int>::iterator mi = bitdb.mapFileUseCount.find(strFile); - if (mi != bitdb.mapFileUseCount.end()) + std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); + if (mi != env->mapFileUseCount.end()) { LogPrint(BCLog::DB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); + env->CloseDb(strFile); + env->CheckpointLSN(strFile); - bitdb.mapFileUseCount.erase(mi++); + env->mapFileUseCount.erase(mi++); LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); ret = true; } @@ -633,3 +637,52 @@ bool CDB::PeriodicFlush(std::string strFile) return ret; } + +bool CWalletDBWrapper::Rewrite(const char* pszSkip) +{ + return CDB::Rewrite(*this, pszSkip); +} + +bool CWalletDBWrapper::Backup(const std::string& strDest) +{ + if (IsDummy()) { + return false; + } + while (true) + { + { + LOCK(env->cs_db); + if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) + { + // Flush log data to the dat file + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(strFile); + + // Copy wallet file + fs::path pathSrc = GetDataDir() / strFile; + fs::path pathDest(strDest); + if (fs::is_directory(pathDest)) + pathDest /= strFile; + + try { + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); + LogPrintf("copied %s to %s\n", strFile, pathDest.string()); + return true; + } catch (const fs::filesystem_error& e) { + LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), e.what()); + return false; + } + } + } + MilliSleep(100); + } + return false; +} + +void CWalletDBWrapper::Flush(bool shutdown) +{ + if (!IsDummy()) { + env->Flush(shutdown); + } +} diff --git a/src/wallet/db.h b/src/wallet/db.h index 9f912f9a1a..3c6870d169 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -72,7 +72,6 @@ public: void CheckpointLSN(const std::string& strFile); void CloseDb(const std::string& strFile); - bool RemoveDb(const std::string& strFile); DbTxn* TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) { @@ -86,6 +85,52 @@ public: extern CDBEnv bitdb; +/** An instance of this class represents one database. + * For BerkeleyDB this is just a (env, strFile) tuple. + **/ +class CWalletDBWrapper +{ + friend class CDB; +public: + /** Create dummy DB handle */ + CWalletDBWrapper(): env(nullptr) + { + } + + /** Create DB handle to real database */ + CWalletDBWrapper(CDBEnv *env_in, const std::string &strFile_in): + env(env_in), strFile(strFile_in) + { + } + + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + bool Rewrite(const char* pszSkip=nullptr); + + /** Back up the entire database to a file. + */ + bool Backup(const std::string& strDest); + + /** Get a name for this database, for debugging etc. + */ + std::string GetName() const { return strFile; } + + /** Make sure all changes are flushed to disk. + */ + void Flush(bool shutdown); + +private: + /** BerkeleyDB specific */ + CDBEnv *env; + std::string strFile; + + /** Return whether this database handle is a dummy for testing. + * Only to be used at a low level, application should ideally not care + * about this. + */ + bool IsDummy() { return env == nullptr; } +}; + /** RAII class that provides access to a Berkeley database */ class CDB @@ -96,18 +141,19 @@ protected: DbTxn* activeTxn; bool fReadOnly; bool fFlushOnClose; + CDBEnv *env; - explicit CDB(const std::string& strFilename, const char* pszMode = "r+", bool fFlushOnCloseIn=true); +public: + explicit CDB(CWalletDBWrapper& dbw, const char* pszMode = "r+", bool fFlushOnCloseIn=true); ~CDB() { Close(); } -public: void Flush(); void Close(); static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)); /* flush the wallet passively (TRY_LOCK) ideal to be called periodically */ - static bool PeriodicFlush(std::string strFile); + static bool PeriodicFlush(CWalletDBWrapper& dbw); /* verifies the database environment */ static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); /* verifies the database file */ @@ -117,7 +163,7 @@ private: CDB(const CDB&); void operator=(const CDB&); -protected: +public: template <typename K, typename T> bool Read(const K& key, T& value) { @@ -134,29 +180,30 @@ protected: Dbt datValue; datValue.set_flags(DB_DBT_MALLOC); int ret = pdb->get(activeTxn, &datKey, &datValue, 0); - memset(datKey.get_data(), 0, datKey.get_size()); - if (datValue.get_data() == NULL) - return false; - - // Unserialize value - try { - CDataStream ssValue((char*)datValue.get_data(), (char*)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); - ssValue >> value; - } catch (const std::exception&) { - return false; + memory_cleanse(datKey.get_data(), datKey.get_size()); + bool success = false; + if (datValue.get_data() != NULL) { + // Unserialize value + try { + CDataStream ssValue((char*)datValue.get_data(), (char*)datValue.get_data() + datValue.get_size(), SER_DISK, CLIENT_VERSION); + ssValue >> value; + success = true; + } catch (const std::exception&) { + // In this case success remains 'false' + } + + // Clear and free memory + memory_cleanse(datValue.get_data(), datValue.get_size()); + free(datValue.get_data()); } - - // Clear and free memory - memset(datValue.get_data(), 0, datValue.get_size()); - free(datValue.get_data()); - return (ret == 0); + return ret == 0 && success; } template <typename K, typename T> bool Write(const K& key, const T& value, bool fOverwrite = true) { if (!pdb) - return false; + return true; if (fReadOnly) assert(!"Write called on database in read-only mode"); @@ -176,8 +223,8 @@ protected: int ret = pdb->put(activeTxn, &datKey, &datValue, (fOverwrite ? 0 : DB_NOOVERWRITE)); // Clear memory in case it was a private key - memset(datKey.get_data(), 0, datKey.get_size()); - memset(datValue.get_data(), 0, datValue.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); + memory_cleanse(datValue.get_data(), datValue.get_size()); return (ret == 0); } @@ -199,7 +246,7 @@ protected: int ret = pdb->del(activeTxn, &datKey, 0); // Clear memory - memset(datKey.get_data(), 0, datKey.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); return (ret == 0 || ret == DB_NOTFOUND); } @@ -219,7 +266,7 @@ protected: int ret = pdb->exists(activeTxn, &datKey, 0); // Clear memory - memset(datKey.get_data(), 0, datKey.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); return (ret == 0); } @@ -262,8 +309,8 @@ protected: ssValue.write((char*)datValue.get_data(), datValue.get_size()); // Clear and free memory - memset(datKey.get_data(), 0, datKey.get_size()); - memset(datValue.get_data(), 0, datValue.get_size()); + memory_cleanse(datKey.get_data(), datKey.get_size()); + memory_cleanse(datValue.get_data(), datValue.get_size()); free(datKey.get_data()); free(datValue.get_data()); return 0; @@ -310,7 +357,7 @@ public: return Write(std::string("version"), nVersion); } - bool static Rewrite(const std::string& strFile, const char* pszSkip = NULL); + bool static Rewrite(CWalletDBWrapper& dbw, const char* pszSkip = NULL); }; #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index fe3871a91d..46ef87b7b1 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -5,6 +5,7 @@ #include "consensus/validation.h" #include "wallet/feebumper.h" #include "wallet/wallet.h" +#include "policy/fees.h" #include "policy/policy.h" #include "policy/rbf.h" #include "validation.h" //for mempool access @@ -23,14 +24,14 @@ int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWallet) { CMutableTransaction txNew(tx); - std::vector<std::pair<const CWalletTx *, unsigned int>> vCoins; + std::vector<CInputCoin> vCoins; // Look up the inputs. We should have already checked that this transaction // IsAllFromMe(ISMINE_SPENDABLE), so every input should already be in our // wallet, with a valid index into the vout array. for (auto& input : tx.vin) { const auto mi = pWallet->mapWallet.find(input.prevout.hash); assert(mi != pWallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size()); - vCoins.emplace_back(&(mi->second), input.prevout.n); + vCoins.emplace_back(CInputCoin(&(mi->second), input.prevout.n)); } if (!pWallet->DummySignTx(txNew, vCoins)) { // This should never happen, because IsAllFromMe(ISMINE_SPENDABLE) @@ -40,7 +41,32 @@ int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *pWal return GetVirtualTransactionSize(txNew); } -CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConfirmTarget, bool specifiedConfirmTarget, CAmount totalFee, bool newTxReplaceable) +bool CFeeBumper::preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx) { + if (pWallet->HasWalletSpend(wtx.GetHash())) { + vErrors.push_back("Transaction has descendants in the wallet"); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return false; + } + + { + LOCK(mempool.cs); + auto it_mp = mempool.mapTx.find(wtx.GetHash()); + if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { + vErrors.push_back("Transaction has descendants in the mempool"); + currentResult = BumpFeeResult::INVALID_PARAMETER; + return false; + } + } + + if (wtx.GetDepthInMainChain() != 0) { + vErrors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); + currentResult = BumpFeeResult::WALLET_ERROR; + return false; + } + return true; +} + +CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConfirmTarget, bool ignoreGlobalPayTxFee, CAmount totalFee, bool newTxReplaceable) : txid(std::move(txidIn)), nOldFee(0), @@ -57,25 +83,7 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConf auto it = pWallet->mapWallet.find(txid); const CWalletTx& wtx = it->second; - if (pWallet->HasWalletSpend(txid)) { - vErrors.push_back("Transaction has descendants in the wallet"); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return; - } - - { - LOCK(mempool.cs); - auto it_mp = mempool.mapTx.find(txid); - if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { - vErrors.push_back("Transaction has descendants in the mempool"); - currentResult = BumpFeeResult::INVALID_PARAMETER; - return; - } - } - - if (wtx.GetDepthInMainChain() != 0) { - vErrors.push_back("Transaction has been mined, or is conflicted with a mined transaction"); - currentResult = BumpFeeResult::WALLET_ERROR; + if (!preconditionChecks(pWallet, wtx)) { return; } @@ -157,15 +165,7 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConf nNewFee = totalFee; nNewFeeRate = CFeeRate(totalFee, maxNewTxSize); } else { - // if user specified a confirm target then don't consider any global payTxFee - if (specifiedConfirmTarget) { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, CAmount(0)); - } - // otherwise use the regular wallet logic to select payTxFee or default confirm target - else { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool); - } - + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, ignoreGlobalPayTxFee); nNewFeeRate = CFeeRate(nNewFee, maxNewTxSize); // New fee rate must be at least old rate + minimum incremental relay rate @@ -213,7 +213,7 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConf // If the output would become dust, discard it (converting the dust to fee) poutput->nValue -= nDelta; - if (poutput->nValue <= poutput->GetDustThreshold(::dustRelayFee)) { + if (poutput->nValue <= GetDustThreshold(*poutput, ::dustRelayFee)) { LogPrint(BCLog::RPC, "Bumping fee and discarding dust output\n"); nNewFee += poutput->nValue; mtx.vout.erase(mtx.vout.begin() + nOutput); @@ -247,6 +247,11 @@ bool CFeeBumper::commit(CWallet *pWallet) } CWalletTx& oldWtx = pWallet->mapWallet[txid]; + // make sure the transaction still has no descendants and hasn't been mined in the meantime + if (!preconditionChecks(pWallet, oldWtx)) { + return false; + } + CWalletTx wtxBumped(pWallet, MakeTransactionRef(std::move(mtx))); // commit/broadcast the tx CReserveKey reservekey(pWallet); diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index 681f55e4e5..fc32316704 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -8,6 +8,7 @@ #include <primitives/transaction.h> class CWallet; +class CWalletTx; class uint256; enum class BumpFeeResult @@ -23,7 +24,7 @@ enum class BumpFeeResult class CFeeBumper { public: - CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, int newConfirmTarget, bool specifiedConfirmTarget, CAmount totalFee, bool newTxReplaceable); + CFeeBumper(const CWallet *pWalletIn, const uint256 txidIn, int newConfirmTarget, bool ignoreGlobalPayTxFee, CAmount totalFee, bool newTxReplaceable); BumpFeeResult getResult() const { return currentResult; } const std::vector<std::string>& getErrors() const { return vErrors; } CAmount getOldFee() const { return nOldFee; } @@ -44,6 +45,8 @@ public: bool commit(CWallet *pWalletNonConst); private: + bool preconditionChecks(const CWallet *pWallet, const CWalletTx& wtx); + const uint256 txid; uint256 bumpedTxid; CMutableTransaction mtx; diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 60666718e3..457e937bb8 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -95,6 +95,8 @@ UniValue importprivkey(const JSONRPCRequest& request) + HelpExampleCli("importprivkey", "\"mykey\"") + "\nImport using a label and without rescan\n" + HelpExampleCli("importprivkey", "\"mykey\" \"testing\" false") + + "\nImport using default blank label and without rescan\n" + + HelpExampleCli("importprivkey", "\"mykey\" \"\" false") + "\nAs a JSON-RPC call\n" + HelpExampleRpc("importprivkey", "\"mykey\", \"testing\", false") ); @@ -154,6 +156,31 @@ UniValue importprivkey(const JSONRPCRequest& request) return NullUniValue; } +UniValue abortrescan(const JSONRPCRequest& request) +{ + CWallet* const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + if (request.fHelp || request.params.size() > 0) + throw std::runtime_error( + "abortrescan\n" + "\nStops current wallet rescan triggered e.g. by an importprivkey call.\n" + "\nExamples:\n" + "\nImport a private key\n" + + HelpExampleCli("importprivkey", "\"mykey\"") + + "\nAbort the running wallet rescan\n" + + HelpExampleCli("abortrescan", "") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("abortrescan", "") + ); + + if (!pwallet->IsScanning() || pwallet->IsAbortingRescan()) return false; + pwallet->AbortRescan(); + return true; +} + void ImportAddress(CWallet*, const CBitcoinAddress& address, const std::string& strLabel); void ImportScript(CWallet* const pwallet, const CScript& script, const std::string& strLabel, bool isRedeemScript) { @@ -330,7 +357,7 @@ UniValue removeprunedfunds(const JSONRPCRequest& request) if (request.fHelp || request.params.size() != 1) throw std::runtime_error( "removeprunedfunds \"txid\"\n" - "\nDeletes the specified transaction from the wallet. Meant for use with pruned wallets and as a companion to importprunedfunds. This will effect wallet balances.\n" + "\nDeletes the specified transaction from the wallet. Meant for use with pruned wallets and as a companion to importprunedfunds. This will affect wallet balances.\n" "\nArguments:\n" "1. \"txid\" (string, required) The hex-encoded id of the transaction you are deleting\n" "\nExamples:\n" @@ -509,14 +536,11 @@ UniValue importwallet(const JSONRPCRequest& request) } file.close(); pwallet->ShowProgress("", 100); // hide progress dialog in GUI - - CBlockIndex *pindex = chainActive.Tip(); - while (pindex && pindex->pprev && pindex->GetBlockTime() > nTimeBegin - TIMESTAMP_WINDOW) - pindex = pindex->pprev; - pwallet->UpdateTimeFirstKey(nTimeBegin); - LogPrintf("Rescanning last %i blocks\n", chainActive.Height() - pindex->nHeight + 1); + CBlockIndex *pindex = chainActive.FindEarliestAtLeast(nTimeBegin - TIMESTAMP_WINDOW); + + LogPrintf("Rescanning last %i blocks\n", pindex ? chainActive.Height() - pindex->nHeight + 1 : 0); pwallet->ScanForWalletTransactions(pindex); pwallet->MarkDirty(); @@ -1104,13 +1128,13 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) if (fRescan && fRunScan && requests.size()) { CBlockIndex* pindex = nLowestTimestamp > minimumTimestamp ? chainActive.FindEarliestAtLeast(std::max<int64_t>(nLowestTimestamp - TIMESTAMP_WINDOW, 0)) : chainActive.Genesis(); - CBlockIndex* scannedRange = nullptr; + CBlockIndex* scanFailed = nullptr; if (pindex) { - scannedRange = pwallet->ScanForWalletTransactions(pindex, true); + scanFailed = pwallet->ScanForWalletTransactions(pindex, true); pwallet->ReacceptWalletTransactions(); } - if (!scannedRange || scannedRange->nHeight > pindex->nHeight) { + if (scanFailed) { std::vector<UniValue> results = response.getValues(); response.clear(); response.setArray(); @@ -1120,12 +1144,23 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) // range, or if the import result already has an error set, let // the result stand unmodified. Otherwise replace the result // with an error message. - if (GetImportTimestamp(request, now) - TIMESTAMP_WINDOW >= scannedRange->GetBlockTimeMax() || results.at(i).exists("error")) { + if (GetImportTimestamp(request, now) - TIMESTAMP_WINDOW > scanFailed->GetBlockTimeMax() || results.at(i).exists("error")) { response.push_back(results.at(i)); } else { UniValue result = UniValue(UniValue::VOBJ); result.pushKV("success", UniValue(false)); - result.pushKV("error", JSONRPCError(RPC_MISC_ERROR, strprintf("Failed to rescan before time %d, transactions may be missing.", scannedRange->GetBlockTimeMax()))); + result.pushKV( + "error", + JSONRPCError( + RPC_MISC_ERROR, + strprintf("Rescan failed for key with creation timestamp %d. There was an error reading a " + "block from time %d, which is after or within %d seconds of key creation, and " + "could contain transactions pertaining to the key. As a result, transactions " + "and coins using this key may not appear in the wallet. This error could be " + "caused by pruning or data corruption (see bitcoind log for details) and could " + "be dealt with by downloading and rescanning the relevant blocks (see -reindex " + "and -rescan options).", + GetImportTimestamp(request, now), scanFailed->GetBlockTimeMax(), TIMESTAMP_WINDOW))); response.push_back(std::move(result)); } ++i; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 2cc3072c16..3b6dfd42ca 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -9,8 +9,11 @@ #include "consensus/validation.h" #include "core_io.h" #include "init.h" +#include "wallet/coincontrol.h" #include "validation.h" #include "net.h" +#include "policy/feerate.h" +#include "policy/fees.h" #include "policy/policy.h" #include "policy/rbf.h" #include "rpc/server.h" @@ -727,6 +730,9 @@ UniValue getbalance(const JSONRPCRequest& request) if (request.params.size() == 0) return ValueFromAmount(pwallet->GetBalance()); + const std::string& account_param = request.params[0].get_str(); + const std::string* account = account_param != "*" ? &account_param : nullptr; + int nMinDepth = 1; if (request.params.size() > 1) nMinDepth = request.params[1].get_int(); @@ -735,41 +741,7 @@ UniValue getbalance(const JSONRPCRequest& request) if(request.params[2].get_bool()) filter = filter | ISMINE_WATCH_ONLY; - if (request.params[0].get_str() == "*") { - // Calculate total balance in a very different way from GetBalance(). - // The biggest difference is that GetBalance() sums up all unspent - // TxOuts paying to the wallet, while this sums up both spent and - // unspent TxOuts paying to the wallet, and then subtracts the values of - // TxIns spending from the wallet. This also has fewer restrictions on - // which unconfirmed transactions are considered trusted. - CAmount nBalance = 0; - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (!CheckFinalTx(wtx) || wtx.GetBlocksToMaturity() > 0 || wtx.GetDepthInMainChain() < 0) - continue; - - CAmount allFee; - std::string strSentAccount; - std::list<COutputEntry> listReceived; - std::list<COutputEntry> listSent; - wtx.GetAmounts(listReceived, listSent, allFee, strSentAccount, filter); - if (wtx.GetDepthInMainChain() >= nMinDepth) - { - BOOST_FOREACH(const COutputEntry& r, listReceived) - nBalance += r.amount; - } - BOOST_FOREACH(const COutputEntry& s, listSent) - nBalance -= s.amount; - nBalance -= allFee; - } - return ValueFromAmount(nBalance); - } - - std::string strAccount = AccountFromValue(request.params[0]); - - CAmount nBalance = pwallet->GetAccountBalance(strAccount, nMinDepth, filter); - - return ValueFromAmount(nBalance); + return ValueFromAmount(pwallet->GetLegacyBalance(filter, nMinDepth, account)); } UniValue getunconfirmedbalance(const JSONRPCRequest &request) @@ -899,7 +871,7 @@ UniValue sendfrom(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); // Check funds - CAmount nBalance = pwallet->GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); + CAmount nBalance = pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth, &strAccount); if (nAmount > nBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Account has insufficient funds"); @@ -1008,7 +980,7 @@ UniValue sendmany(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); // Check funds - CAmount nBalance = pwallet->GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); + CAmount nBalance = pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth, &strAccount); if (totalAmount > nBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Account has insufficient funds"); @@ -2076,7 +2048,7 @@ UniValue walletpassphrase(const JSONRPCRequest& request) int64_t nSleepTime = request.params[1].get_int64(); pwallet->nRelockTime = GetTime() + nSleepTime; - RPCRunLater(strprintf("lockwallet(%s)", pwallet->strWalletFile), boost::bind(LockWallet, pwallet), nSleepTime); + RPCRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), boost::bind(LockWallet, pwallet), nSleepTime); return NullUniValue; } @@ -2498,22 +2470,29 @@ UniValue listunspent(const JSONRPCRequest& request) return NullUniValue; } - if (request.fHelp || request.params.size() > 4) + if (request.fHelp || request.params.size() > 5) throw std::runtime_error( - "listunspent ( minconf maxconf [\"addresses\",...] [include_unsafe] )\n" + "listunspent ( minconf maxconf [\"addresses\",...] [include_unsafe] [query_options])\n" "\nReturns array of unspent transaction outputs\n" "with between minconf and maxconf (inclusive) confirmations.\n" "Optionally filter to only include txouts paid to specified addresses.\n" "\nArguments:\n" "1. minconf (numeric, optional, default=1) The minimum confirmations to filter\n" "2. maxconf (numeric, optional, default=9999999) The maximum confirmations to filter\n" - "3. \"addresses\" (string) A json array of bitcoin addresses to filter\n" + "3. \"addresses\" (string) A json array of bitcoin addresses to filter\n" " [\n" - " \"address\" (string) bitcoin address\n" + " \"address\" (string) bitcoin address\n" " ,...\n" " ]\n" "4. include_unsafe (bool, optional, default=true) Include outputs that are not safe to spend\n" " See description of \"safe\" attribute below.\n" + "5. query_options (json, optional) JSON with query options\n" + " {\n" + " \"minimumAmount\" (numeric or string, default=0) Minimum value of each UTXO in " + CURRENCY_UNIT + "\n" + " \"maximumAmount\" (numeric or string, default=unlimited) Maximum value of each UTXO in " + CURRENCY_UNIT + "\n" + " \"maximumCount\" (numeric or string, default=unlimited) Maximum number of UTXOs\n" + " \"minimumSumAmount\" (numeric or string, default=unlimited) Minimum sum value of all UTXOs in " + CURRENCY_UNIT + "\n" + " }\n" "\nResult\n" "[ (array of json object)\n" " {\n" @@ -2538,6 +2517,8 @@ UniValue listunspent(const JSONRPCRequest& request) + HelpExampleCli("listunspent", "") + HelpExampleCli("listunspent", "6 9999999 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + HelpExampleRpc("listunspent", "6, 9999999 \"[\\\"1PGFqEzfmQch1gKD3ra4k18PNj3tTUUSqg\\\",\\\"1LtvqCaApEdUGFkpKMM4MstjcaL4dKg8SP\\\"]\"") + + HelpExampleCli("listunspent", "6 9999999 '[]' true '{ \"minimumAmount\": 0.005 }'") + + HelpExampleRpc("listunspent", "6, 9999999, [] , true, { \"minimumAmount\": 0.005 } ") ); int nMinDepth = 1; @@ -2573,15 +2554,34 @@ UniValue listunspent(const JSONRPCRequest& request) include_unsafe = request.params[3].get_bool(); } + CAmount nMinimumAmount = 0; + CAmount nMaximumAmount = MAX_MONEY; + CAmount nMinimumSumAmount = MAX_MONEY; + uint64_t nMaximumCount = 0; + + if (request.params.size() > 4) { + const UniValue& options = request.params[4].get_obj(); + + if (options.exists("minimumAmount")) + nMinimumAmount = AmountFromValue(options["minimumAmount"]); + + if (options.exists("maximumAmount")) + nMaximumAmount = AmountFromValue(options["maximumAmount"]); + + if (options.exists("minimumSumAmount")) + nMinimumSumAmount = AmountFromValue(options["minimumSumAmount"]); + + if (options.exists("maximumCount")) + nMaximumCount = options["maximumCount"].get_int64(); + } + UniValue results(UniValue::VARR); std::vector<COutput> vecOutputs; assert(pwallet != NULL); LOCK2(cs_main, pwallet->cs_wallet); - pwallet->AvailableCoins(vecOutputs, !include_unsafe, NULL, true); - BOOST_FOREACH(const COutput& out, vecOutputs) { - if (out.nDepth < nMinDepth || out.nDepth > nMaxDepth) - continue; + pwallet->AvailableCoins(vecOutputs, !include_unsafe, NULL, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount, nMinDepth, nMaxDepth); + BOOST_FOREACH(const COutput& out, vecOutputs) { CTxDestination address; const CScript& scriptPubKey = out.tx->tx->vout[out.i].scriptPubKey; bool fValidAddress = ExtractDestination(scriptPubKey, address); @@ -2797,7 +2797,7 @@ UniValue bumpfee(const JSONRPCRequest& request) "The command will fail if the wallet or mempool contains a transaction that spends one of T's outputs.\n" "By default, the new fee will be calculated automatically using estimatefee.\n" "The user can specify a confirmation target for estimatefee.\n" - "Alternatively, the user can specify totalFee, or use RPC setpaytxfee to set a higher fee rate.\n" + "Alternatively, the user can specify totalFee, or use RPC settxfee to set a higher fee rate.\n" "At a minimum, the new fee rate must be high enough to pay an additional new relay fee (incrementalfee\n" "returned by getnetworkinfo) to enter the node's mempool.\n" "\nArguments:\n" @@ -2834,7 +2834,7 @@ UniValue bumpfee(const JSONRPCRequest& request) hash.SetHex(request.params[0].get_str()); // optional parameters - bool specifiedConfirmTarget = false; + bool ignoreGlobalPayTxFee = false; int newConfirmTarget = nTxConfirmTarget; CAmount totalFee = 0; bool replaceable = true; @@ -2851,7 +2851,10 @@ UniValue bumpfee(const JSONRPCRequest& request) if (options.exists("confTarget") && options.exists("totalFee")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "confTarget and totalFee options should not both be set. Please provide either a confirmation target for fee estimation or an explicit total fee for the transaction."); } else if (options.exists("confTarget")) { - specifiedConfirmTarget = true; + // If the user has explicitly set a confTarget in this rpc call, + // then override the default logic that uses the global payTxFee + // instead of the confirmation target. + ignoreGlobalPayTxFee = true; newConfirmTarget = options["confTarget"].get_int(); if (newConfirmTarget <= 0) { // upper-bound will be checked by estimatefee/smartfee throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid confTarget (cannot be <= 0)"); @@ -2871,7 +2874,7 @@ UniValue bumpfee(const JSONRPCRequest& request) LOCK2(cs_main, pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); - CFeeBumper feeBump(pwallet, hash, newConfirmTarget, specifiedConfirmTarget, totalFee, replaceable); + CFeeBumper feeBump(pwallet, hash, newConfirmTarget, ignoreGlobalPayTxFee, totalFee, replaceable); BumpFeeResult res = feeBump.getResult(); if (res != BumpFeeResult::OK) { @@ -2909,11 +2912,12 @@ UniValue bumpfee(const JSONRPCRequest& request) UniValue errors(UniValue::VARR); for (const std::string& err: feeBump.getErrors()) errors.push_back(err); - result.push_back(errors); + result.push_back(Pair("errors", errors)); return result; } +extern UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp extern UniValue importprivkey(const JSONRPCRequest& request); extern UniValue importaddress(const JSONRPCRequest& request); @@ -2930,6 +2934,7 @@ static const CRPCCommand commands[] = { "rawtransactions", "fundrawtransaction", &fundrawtransaction, false, {"hexstring","options"} }, { "hidden", "resendwallettransactions", &resendwallettransactions, true, {} }, { "wallet", "abandontransaction", &abandontransaction, false, {"txid"} }, + { "wallet", "abortrescan", &abortrescan, false, {} }, { "wallet", "addmultisigaddress", &addmultisigaddress, true, {"nrequired","keys","account"} }, { "wallet", "addwitnessaddress", &addwitnessaddress, true, {"address"} }, { "wallet", "backupwallet", &backupwallet, true, {"destination"} }, @@ -2962,7 +2967,7 @@ static const CRPCCommand commands[] = { "wallet", "listreceivedbyaddress", &listreceivedbyaddress, false, {"minconf","include_empty","include_watchonly"} }, { "wallet", "listsinceblock", &listsinceblock, false, {"blockhash","target_confirmations","include_watchonly"} }, { "wallet", "listtransactions", &listtransactions, false, {"account","count","skip","include_watchonly"} }, - { "wallet", "listunspent", &listunspent, false, {"minconf","maxconf","addresses","include_unsafe"} }, + { "wallet", "listunspent", &listunspent, false, {"minconf","maxconf","addresses","include_unsafe","query_options"} }, { "wallet", "lockunspent", &lockunspent, true, {"unlock","transactions"} }, { "wallet", "move", &movecmd, false, {"fromaccount","toaccount","amount","minconf","comment"} }, { "wallet", "sendfrom", &sendfrom, false, {"fromaccount","toaddress","amount","minconf","comment","comment_to"} }, diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index a76db37617..1989bf8d9b 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -14,7 +14,8 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName): bitdb.MakeMock(); bool fFirstRun; - pwalletMain = new CWallet("wallet_test.dat"); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); + pwalletMain = new CWallet(std::move(dbw)); pwalletMain->LoadWallet(fFirstRun); RegisterValidationInterface(pwalletMain); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 67e5e90224..b30748d66b 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -9,6 +9,7 @@ #include <utility> #include <vector> +#include "consensus/validation.h" #include "rpc/server.h" #include "test/test_bitcoin.h" #include "validation.h" @@ -19,6 +20,8 @@ #include <univalue.h> extern UniValue importmulti(const JSONRPCRequest& request); +extern UniValue dumpwallet(const JSONRPCRequest& request); +extern UniValue importwallet(const JSONRPCRequest& request); // how many times to run all the tests to have a chance to catch errors that only show up with particular random shuffles #define RUN_TESTS 100 @@ -29,7 +32,7 @@ extern UniValue importmulti(const JSONRPCRequest& request); std::vector<std::unique_ptr<CWalletTx>> wtxn; -typedef std::set<std::pair<const CWalletTx*,unsigned int> > CoinSet; +typedef std::set<CInputCoin> CoinSet; BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) @@ -364,6 +367,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) LOCK(cs_main); // Cap last block file size, and mine new block in a new block file. + CBlockIndex* const nullBlock = nullptr; CBlockIndex* oldTip = chainActive.Tip(); GetBlockFileInfo(oldTip->GetBlockPos().nFile)->nSize = MAX_BLOCKFILE_SIZE; CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); @@ -375,7 +379,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CWallet wallet; LOCK(wallet.cs_wallet); wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); - BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip)); + BOOST_CHECK_EQUAL(nullBlock, wallet.ScanForWalletTransactions(oldTip)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 100 * COIN); } @@ -389,7 +393,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CWallet wallet; LOCK(wallet.cs_wallet); wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); - BOOST_CHECK_EQUAL(newTip, wallet.ScanForWalletTransactions(oldTip)); + BOOST_CHECK_EQUAL(oldTip, wallet.ScanForWalletTransactions(oldTip)); BOOST_CHECK_EQUAL(wallet.GetImmatureBalance(), 50 * COIN); } @@ -413,7 +417,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CKey futureKey; futureKey.MakeNewKey(true); key.pushKV("scriptPubKey", HexStr(GetScriptForRawPubKey(futureKey.GetPubKey()))); - key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW); + key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1); key.pushKV("internal", UniValue(true)); keys.push_back(key); JSONRPCRequest request; @@ -421,11 +425,79 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) request.params.push_back(keys); UniValue response = importmulti(request); - BOOST_CHECK_EQUAL(response.write(), strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Failed to rescan before time %d, transactions may be missing.\"}},{\"success\":true}]", newTip->GetBlockTimeMax())); + BOOST_CHECK_EQUAL(response.write(), + strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Rescan failed for key with creation " + "timestamp %d. There was an error reading a block from time %d, which is after or within %d " + "seconds of key creation, and could contain transactions pertaining to the key. As a result, " + "transactions and coins using this key may not appear in the wallet. This error could be caused " + "by pruning or data corruption (see bitcoind log for details) and could be dealt with by " + "downloading and rescanning the relevant blocks (see -reindex and -rescan " + "options).\"}},{\"success\":true}]", + 0, oldTip->GetBlockTimeMax(), TIMESTAMP_WINDOW)); ::pwalletMain = backup; } } +// Verify importwallet RPC starts rescan at earliest block with timestamp +// greater or equal than key birthday. Previously there was a bug where +// importwallet RPC would start the scan at the latest block with timestamp less +// than or equal to key birthday. +BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) +{ + CWallet *pwalletMainBackup = ::pwalletMain; + LOCK(cs_main); + + // Create two blocks with same timestamp to verify that importwallet rescan + // will pick up both blocks, not just the first. + const int64_t BLOCK_TIME = chainActive.Tip()->GetBlockTimeMax() + 5; + SetMockTime(BLOCK_TIME); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + + // Set key birthday to block time increased by the timestamp window, so + // rescan will start at the block time. + const int64_t KEY_TIME = BLOCK_TIME + TIMESTAMP_WINDOW; + SetMockTime(KEY_TIME); + coinbaseTxns.emplace_back(*CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + + // Import key into wallet and call dumpwallet to create backup file. + { + CWallet wallet; + LOCK(wallet.cs_wallet); + wallet.mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; + wallet.AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + + JSONRPCRequest request; + request.params.setArray(); + request.params.push_back("wallet.backup"); + ::pwalletMain = &wallet; + ::dumpwallet(request); + } + + // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME + // were scanned, and no prior blocks were scanned. + { + CWallet wallet; + + JSONRPCRequest request; + request.params.setArray(); + request.params.push_back("wallet.backup"); + ::pwalletMain = &wallet; + ::importwallet(request); + + BOOST_CHECK_EQUAL(wallet.mapWallet.size(), 3); + BOOST_CHECK_EQUAL(coinbaseTxns.size(), 103); + for (size_t i = 0; i < coinbaseTxns.size(); ++i) { + bool found = wallet.GetWalletTx(coinbaseTxns[i].GetHash()); + bool expected = i >= 100; + BOOST_CHECK_EQUAL(found, expected); + } + } + + SetMockTime(0); + ::pwalletMain = pwalletMainBackup; +} + // Check that GetImmatureCredit() returns a newly calculated value instead of // the cached value after a MarkDirty() call. // @@ -504,4 +576,104 @@ BOOST_AUTO_TEST_CASE(ComputeTimeSmart) SetMockTime(0); } +BOOST_AUTO_TEST_CASE(LoadReceiveRequests) +{ + CTxDestination dest = CKeyID(); + pwalletMain->AddDestData(dest, "misc", "val_misc"); + pwalletMain->AddDestData(dest, "rr0", "val_rr0"); + pwalletMain->AddDestData(dest, "rr1", "val_rr1"); + + auto values = pwalletMain->GetDestValues("rr"); + BOOST_CHECK_EQUAL(values.size(), 2); + BOOST_CHECK_EQUAL(values[0], "val_rr0"); + BOOST_CHECK_EQUAL(values[1], "val_rr1"); +} + +class ListCoinsTestingSetup : public TestChain100Setup +{ +public: + ListCoinsTestingSetup() + { + CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + ::bitdb.MakeMock(); + wallet.reset(new CWallet(std::unique_ptr<CWalletDBWrapper>(new CWalletDBWrapper(&bitdb, "wallet_test.dat")))); + bool firstRun; + wallet->LoadWallet(firstRun); + LOCK(wallet->cs_wallet); + wallet->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + wallet->ScanForWalletTransactions(chainActive.Genesis()); + } + + ~ListCoinsTestingSetup() + { + wallet.reset(); + ::bitdb.Flush(true); + ::bitdb.Reset(); + } + + CWalletTx& AddTx(CRecipient recipient) + { + CWalletTx wtx; + CReserveKey reservekey(wallet.get()); + CAmount fee; + int changePos = -1; + std::string error; + BOOST_CHECK(wallet->CreateTransaction({recipient}, wtx, reservekey, fee, changePos, error)); + CValidationState state; + BOOST_CHECK(wallet->CommitTransaction(wtx, reservekey, nullptr, state)); + auto it = wallet->mapWallet.find(wtx.GetHash()); + BOOST_CHECK(it != wallet->mapWallet.end()); + CreateAndProcessBlock({CMutableTransaction(*it->second.tx)}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + it->second.SetMerkleBranch(chainActive.Tip(), 1); + return it->second; + } + + std::unique_ptr<CWallet> wallet; +}; + +BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) +{ + std::string coinbaseAddress = coinbaseKey.GetPubKey().GetID().ToString(); + LOCK(wallet->cs_wallet); + + // Confirm ListCoins initially returns 1 coin grouped under coinbaseKey + // address. + auto list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 1); + + // Check initial balance from one mature coinbase transaction. + BOOST_CHECK_EQUAL(50 * COIN, wallet->GetAvailableBalance()); + + // Add a transaction creating a change address, and confirm ListCoins still + // returns the coin associated with the change address underneath the + // coinbaseKey pubkey, even though the change address has a different + // pubkey. + AddTx(CRecipient{GetScriptForRawPubKey({}), 1 * COIN, false /* subtract fee */}); + list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 2); + + // Lock both coins. Confirm number of available coins drops to 0. + std::vector<COutput> available; + wallet->AvailableCoins(available); + BOOST_CHECK_EQUAL(available.size(), 2); + for (const auto& group : list) { + for (const auto& coin : group.second) { + wallet->LockCoin(COutPoint(coin.tx->GetHash(), coin.i)); + } + } + wallet->AvailableCoins(available); + BOOST_CHECK_EQUAL(available.size(), 0); + + // Confirm ListCoins still returns same result as before, despite coins + // being locked. + list = wallet->ListCoins(); + BOOST_CHECK_EQUAL(list.size(), 1); + BOOST_CHECK_EQUAL(boost::get<CKeyID>(list.begin()->first).ToString(), coinbaseAddress); + BOOST_CHECK_EQUAL(list.begin()->second.size(), 2); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index b0aa743422..3e000d2a9d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -16,6 +16,7 @@ #include "keystore.h" #include "validation.h" #include "net.h" +#include "policy/fees.h" #include "policy/policy.h" #include "policy/rbf.h" #include "primitives/block.h" @@ -65,10 +66,10 @@ const uint256 CMerkleTx::ABANDON_HASH(uint256S("00000000000000000000000000000000 struct CompareValueOnly { - bool operator()(const std::pair<CAmount, std::pair<const CWalletTx*, unsigned int> >& t1, - const std::pair<CAmount, std::pair<const CWalletTx*, unsigned int> >& t2) const + bool operator()(const CInputCoin& t1, + const CInputCoin& t2) const { - return t1.first < t2.first; + return t1.txout.nValue < t2.txout.nValue; } }; @@ -161,7 +162,7 @@ void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret, bool inter secret = childKey.key; metadata.hdMasterKeyID = hdChain.masterKeyID; // update the chain model in the database - if (!CWalletDB(strWalletFile).WriteHDChain(hdChain)) + if (!CWalletDB(*dbw).WriteHDChain(hdChain)) throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); } @@ -180,10 +181,8 @@ bool CWallet::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) if (HaveWatchOnly(script)) RemoveWatchOnly(script); - if (!fFileBacked) - return true; if (!IsCrypted()) { - return CWalletDB(strWalletFile).WriteKey(pubkey, + return CWalletDB(*dbw).WriteKey(pubkey, secret.GetPrivKey(), mapKeyMetadata[pubkey.GetID()]); } @@ -195,8 +194,6 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, { if (!CCryptoKeyStore::AddCryptedKey(vchPubKey, vchCryptedSecret)) return false; - if (!fFileBacked) - return true; { LOCK(cs_wallet); if (pwalletdbEncryption) @@ -204,7 +201,7 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, vchCryptedSecret, mapKeyMetadata[vchPubKey.GetID()]); else - return CWalletDB(strWalletFile).WriteCryptedKey(vchPubKey, + return CWalletDB(*dbw).WriteCryptedKey(vchPubKey, vchCryptedSecret, mapKeyMetadata[vchPubKey.GetID()]); } @@ -240,9 +237,7 @@ bool CWallet::AddCScript(const CScript& redeemScript) { if (!CCryptoKeyStore::AddCScript(redeemScript)) return false; - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteCScript(Hash160(redeemScript), redeemScript); + return CWalletDB(*dbw).WriteCScript(Hash160(redeemScript), redeemScript); } bool CWallet::LoadCScript(const CScript& redeemScript) @@ -268,9 +263,7 @@ bool CWallet::AddWatchOnly(const CScript& dest) const CKeyMetadata& meta = mapKeyMetadata[CScriptID(dest)]; UpdateTimeFirstKey(meta.nCreateTime); NotifyWatchonlyChanged(true); - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteWatchOnly(dest, meta); + return CWalletDB(*dbw).WriteWatchOnly(dest, meta); } bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) @@ -286,9 +279,8 @@ bool CWallet::RemoveWatchOnly(const CScript &dest) return false; if (!HaveWatchOnly()) NotifyWatchonlyChanged(false); - if (fFileBacked) - if (!CWalletDB(strWalletFile).EraseWatchOnly(dest)) - return false; + if (!CWalletDB(*dbw).EraseWatchOnly(dest)) + return false; return true; } @@ -353,7 +345,7 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, return false; if (!crypter.Encrypt(_vMasterKey, pMasterKey.second.vchCryptedKey)) return false; - CWalletDB(strWalletFile).WriteMasterKey(pMasterKey.first, pMasterKey.second); + CWalletDB(*dbw).WriteMasterKey(pMasterKey.first, pMasterKey.second); if (fWasLocked) Lock(); return true; @@ -366,7 +358,7 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, void CWallet::SetBestChain(const CBlockLocator& loc) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); walletdb.WriteBestBlock(loc); } @@ -385,9 +377,8 @@ bool CWallet::SetMinVersion(enum WalletFeature nVersion, CWalletDB* pwalletdbIn, if (nVersion > nWalletMaxVersion) nWalletMaxVersion = nVersion; - if (fFileBacked) { - CWalletDB* pwalletdb = pwalletdbIn ? pwalletdbIn : new CWalletDB(strWalletFile); + CWalletDB* pwalletdb = pwalletdbIn ? pwalletdbIn : new CWalletDB(*dbw); if (nWalletVersion > 40000) pwalletdb->WriteMinVersion(nWalletVersion); if (!pwalletdbIn) @@ -441,7 +432,7 @@ bool CWallet::HasWalletSpend(const uint256& txid) const void CWallet::Flush(bool shutdown) { - bitdb.Flush(shutdown); + dbw->Flush(shutdown); } bool CWallet::Verify() @@ -594,24 +585,19 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) { LOCK(cs_wallet); mapMasterKeys[++nMasterKeyMaxID] = kMasterKey; - if (fFileBacked) - { - assert(!pwalletdbEncryption); - pwalletdbEncryption = new CWalletDB(strWalletFile); - if (!pwalletdbEncryption->TxnBegin()) { - delete pwalletdbEncryption; - pwalletdbEncryption = NULL; - return false; - } - pwalletdbEncryption->WriteMasterKey(nMasterKeyMaxID, kMasterKey); + assert(!pwalletdbEncryption); + pwalletdbEncryption = new CWalletDB(*dbw); + if (!pwalletdbEncryption->TxnBegin()) { + delete pwalletdbEncryption; + pwalletdbEncryption = NULL; + return false; } + pwalletdbEncryption->WriteMasterKey(nMasterKeyMaxID, kMasterKey); if (!EncryptKeys(_vMasterKey)) { - if (fFileBacked) { - pwalletdbEncryption->TxnAbort(); - delete pwalletdbEncryption; - } + pwalletdbEncryption->TxnAbort(); + delete pwalletdbEncryption; // We now probably have half of our keys encrypted in memory, and half not... // die and let the user reload the unencrypted wallet. assert(false); @@ -620,30 +606,24 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) // Encryption was introduced in version 0.4.0 SetMinVersion(FEATURE_WALLETCRYPT, pwalletdbEncryption, true); - if (fFileBacked) - { - if (!pwalletdbEncryption->TxnCommit()) { - delete pwalletdbEncryption; - // We now have keys encrypted in memory, but not on disk... - // die to avoid confusion and let the user reload the unencrypted wallet. - assert(false); - } - + if (!pwalletdbEncryption->TxnCommit()) { delete pwalletdbEncryption; - pwalletdbEncryption = NULL; + // We now have keys encrypted in memory, but not on disk... + // die to avoid confusion and let the user reload the unencrypted wallet. + assert(false); } + delete pwalletdbEncryption; + pwalletdbEncryption = NULL; + Lock(); Unlock(strWalletPassphrase); // if we are using HD, replace the HD master key (seed) with a new one if (IsHDEnabled()) { - CKey key; - CPubKey masterPubKey = GenerateNewHDMasterKey(); - // preserve the old chains version to not break backward compatibility - CHDChain oldChain = GetHDChain(); - if (!SetHDMasterKey(masterPubKey, &oldChain)) + if (!SetHDMasterKey(GenerateNewHDMasterKey())) { return false; + } } NewKeyPool(); @@ -651,7 +631,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) // Need to completely rewrite the wallet file; if we don't, bdb might keep // bits of the unencrypted private key in slack space in the database file. - CDB::Rewrite(strWalletFile); + dbw->Rewrite(); } NotifyStatusChanged(this); @@ -662,7 +642,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) DBErrors CWallet::ReorderTransactions() { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); // Old wallets didn't have any defined order for transactions // Probably a bad idea to change the output of this @@ -743,14 +723,14 @@ int64_t CWallet::IncOrderPosNext(CWalletDB *pwalletdb) if (pwalletdb) { pwalletdb->WriteOrderPosNext(nOrderPosNext); } else { - CWalletDB(strWalletFile).WriteOrderPosNext(nOrderPosNext); + CWalletDB(*dbw).WriteOrderPosNext(nOrderPosNext); } return nRet; } bool CWallet::AccountMove(std::string strFrom, std::string strTo, CAmount nAmount, std::string strComment) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); if (!walletdb.TxnBegin()) return false; @@ -784,7 +764,7 @@ bool CWallet::AccountMove(std::string strFrom, std::string strTo, CAmount nAmoun bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bForceNew) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); CAccount account; walletdb.ReadAccount(strAccount, account); @@ -845,7 +825,7 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) wtx.mapValue["replaced_by_txid"] = newHash.ToString(); - CWalletDB walletdb(strWalletFile, "r+"); + CWalletDB walletdb(*dbw, "r+"); bool success = true; if (!walletdb.WriteTx(wtx)) { @@ -862,7 +842,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile, "r+", fFlushOnClose); + CWalletDB walletdb(*dbw, "r+", fFlushOnClose); uint256 hash = wtxIn.GetHash(); @@ -957,9 +937,9 @@ bool CWallet::LoadToWallet(const CWalletTx& wtxIn) /** * Add a transaction to the wallet, or update it. pIndex and posInBlock should * be set when the transaction was known to be included in a block. When - * posInBlock = SYNC_TRANSACTION_NOT_IN_BLOCK (-1) , then wallet state is not - * updated in AddToWallet, but notifications happen and cached balances are - * marked dirty. + * pIndex == NULL, then wallet state is not updated in AddToWallet, but + * notifications happen and cached balances are marked dirty. + * * If fUpdate is true, existing transactions will be updated. * TODO: One exception to this is that the abandoned state is cleared under the * assumption that any further notification of a transaction that was considered @@ -973,7 +953,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockI { AssertLockHeld(cs_wallet); - if (posInBlock != -1) { + if (pIndex != NULL) { BOOST_FOREACH(const CTxIn& txin, tx.vin) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(txin.prevout); while (range.first != range.second) { @@ -993,7 +973,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockI CWalletTx wtx(this, ptx); // Get merkle branch if transaction was found in a block - if (posInBlock != -1) + if (pIndex != NULL) wtx.SetMerkleBranch(pIndex, posInBlock); return AddToWallet(wtx, false); @@ -1002,11 +982,18 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockI return false; } +bool CWallet::TransactionCanBeAbandoned(const uint256& hashTx) const +{ + LOCK2(cs_main, cs_wallet); + const CWalletTx* wtx = GetWalletTx(hashTx); + return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain() <= 0 && !wtx->InMempool(); +} + bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK2(cs_main, cs_wallet); - CWalletDB walletdb(strWalletFile, "r+"); + CWalletDB walletdb(*dbw, "r+"); std::set<uint256> todo; std::set<uint256> done; @@ -1078,7 +1065,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) return; // Do not flush the wallet here for performance reasons - CWalletDB walletdb(strWalletFile, "r+", false); + CWalletDB walletdb(*dbw, "r+", false); std::set<uint256> todo; std::set<uint256> done; @@ -1118,10 +1105,10 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) } } -void CWallet::SyncTransaction(const CTransactionRef& ptx, const CBlockIndex *pindexBlockConnected, int posInBlock) { +void CWallet::SyncTransaction(const CTransactionRef& ptx, const CBlockIndex *pindex, int posInBlock) { const CTransaction& tx = *ptx; - if (!AddToWalletIfInvolvingMe(ptx, pindexBlockConnected, posInBlock, true)) + if (!AddToWalletIfInvolvingMe(ptx, pindex, posInBlock, true)) return; // Not one of ours // If a transaction changes 'conflicted' state, that changes the balance @@ -1136,21 +1123,21 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, const CBlockIndex *pin void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { LOCK2(cs_main, cs_wallet); - SyncTransaction(ptx, NULL, -1); + SyncTransaction(ptx); } void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { LOCK2(cs_main, cs_wallet); - // TODO: Tempoarily ensure that mempool removals are notified before + // TODO: Temporarily ensure that mempool removals are notified before // connected transactions. This shouldn't matter, but the abandoned // state of transactions in our wallet is currently cleared when we // receive another notification and there is a race condition where // notification of a connected conflict might cause an outside process - // to abandon a transaction and then have it inadvertantly cleared by + // to abandon a transaction and then have it inadvertently cleared by // the notification that the conflicted transaction was evicted. for (const CTransactionRef& ptx : vtxConflicted) { - SyncTransaction(ptx, NULL, -1); + SyncTransaction(ptx); } for (size_t i = 0; i < pblock->vtx.size(); i++) { SyncTransaction(pblock->vtx[i], pindex, i); @@ -1161,7 +1148,7 @@ void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { LOCK2(cs_main, cs_wallet); for (const CTransactionRef& ptx : pblock->vtx) { - SyncTransaction(ptx, NULL, -1); + SyncTransaction(ptx); } } @@ -1341,17 +1328,14 @@ CPubKey CWallet::GenerateNewHDMasterKey() return pubkey; } -bool CWallet::SetHDMasterKey(const CPubKey& pubkey, CHDChain *possibleOldChain) +bool CWallet::SetHDMasterKey(const CPubKey& pubkey) { LOCK(cs_wallet); // store the keyid (hash160) together with // the child index counter in the database // as a hdchain object CHDChain newHdChain; - if (possibleOldChain) { - // preserve the old chains version - newHdChain.nVersion = possibleOldChain->nVersion; - } + newHdChain.nVersion = CanSupportFeature(FEATURE_HD_SPLIT) ? CHDChain::VERSION_HD_CHAIN_SPLIT : CHDChain::VERSION_HD_BASE; newHdChain.masterKeyID = pubkey.GetID(); SetHDChain(newHdChain, false); @@ -1361,7 +1345,7 @@ bool CWallet::SetHDMasterKey(const CPubKey& pubkey, CHDChain *possibleOldChain) bool CWallet::SetHDChain(const CHDChain& chain, bool memonly) { LOCK(cs_wallet); - if (!memonly && !CWalletDB(strWalletFile).WriteHDChain(chain)) + if (!memonly && !CWalletDB(*dbw).WriteHDChain(chain)) throw std::runtime_error(std::string(__func__) + ": writing chain failed"); hdChain = chain; @@ -1474,59 +1458,26 @@ void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, } -void CWalletTx::GetAccountAmounts(const std::string& strAccount, CAmount& nReceived, - CAmount& nSent, CAmount& nFee, const isminefilter& filter) const -{ - nReceived = nSent = nFee = 0; - - CAmount allFee; - std::string strSentAccount; - std::list<COutputEntry> listReceived; - std::list<COutputEntry> listSent; - GetAmounts(listReceived, listSent, allFee, strSentAccount, filter); - - if (strAccount == strSentAccount) - { - BOOST_FOREACH(const COutputEntry& s, listSent) - nSent += s.amount; - nFee = allFee; - } - { - LOCK(pwallet->cs_wallet); - BOOST_FOREACH(const COutputEntry& r, listReceived) - { - if (pwallet->mapAddressBook.count(r.destination)) - { - std::map<CTxDestination, CAddressBookData>::const_iterator mi = pwallet->mapAddressBook.find(r.destination); - if (mi != pwallet->mapAddressBook.end() && (*mi).second.name == strAccount) - nReceived += r.amount; - } - else if (strAccount.empty()) - { - nReceived += r.amount; - } - } - } -} - /** * Scan the block chain (starting in pindexStart) for transactions * from or to us. If fUpdate is true, found transactions that already * exist in the wallet will be updated. * - * Returns pointer to the first block in the last contiguous range that was - * successfully scanned. - * + * Returns null if scan was successful. Otherwise, if a complete rescan was not + * possible (due to pruning or corruption), returns pointer to the most recent + * block that could not be scanned. */ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate) { - CBlockIndex* ret = nullptr; int64_t nNow = GetTime(); const CChainParams& chainParams = Params(); CBlockIndex* pindex = pindexStart; + CBlockIndex* ret = nullptr; { LOCK2(cs_main, cs_wallet); + fAbortRescan = false; + fScanningWallet = true; // no need to read and scan block, if block was created before // our wallet birthday (as adjusted for block time variability) @@ -1536,29 +1487,31 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup double dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex); double dProgressTip = GuessVerificationProgress(chainParams.TxData(), chainActive.Tip()); - while (pindex) + while (pindex && !fAbortRescan) { if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((GuessVerificationProgress(chainParams.TxData(), pindex) - dProgressStart) / (dProgressTip - dProgressStart) * 100)))); + if (GetTime() >= nNow + 60) { + nNow = GetTime(); + LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); + } CBlock block; if (ReadBlockFromDisk(block, pindex, Params().GetConsensus())) { for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { AddToWalletIfInvolvingMe(block.vtx[posInBlock], pindex, posInBlock, fUpdate); } - if (!ret) { - ret = pindex; - } } else { - ret = nullptr; + ret = pindex; } pindex = chainActive.Next(pindex); - if (GetTime() >= nNow + 60) { - nNow = GetTime(); - LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); - } + } + if (pindex && fAbortRescan) { + LogPrintf("Rescan aborted at block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); } ShowProgress(_("Rescanning..."), 100); // hide progress dialog in GUI + + fScanningWallet = false; } return ret; } @@ -1831,8 +1784,8 @@ bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { CMutableTransaction tx1 = *this->tx; CMutableTransaction tx2 = *_tx.tx; - for (unsigned int i = 0; i < tx1.vin.size(); i++) tx1.vin[i].scriptSig = CScript(); - for (unsigned int i = 0; i < tx2.vin.size(); i++) tx2.vin[i].scriptSig = CScript(); + for (auto& txin : tx1.vin) txin.scriptSig = CScript(); + for (auto& txin : tx2.vin) txin.scriptSig = CScript(); return CTransaction(tx1) == CTransaction(tx2); } @@ -1984,12 +1937,73 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const return nTotal; } -void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const CCoinControl *coinControl, bool fIncludeZeroValue) const +// Calculate total balance in a different way from GetBalance. The biggest +// difference is that GetBalance sums up all unspent TxOuts paying to the +// wallet, while this sums up both spent and unspent TxOuts paying to the +// wallet, and then subtracts the values of TxIns spending from the wallet. This +// also has fewer restrictions on which unconfirmed transactions are considered +// trusted. +CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth, const std::string* account) const +{ + LOCK2(cs_main, cs_wallet); + + CAmount balance = 0; + for (const auto& entry : mapWallet) { + const CWalletTx& wtx = entry.second; + const int depth = wtx.GetDepthInMainChain(); + if (depth < 0 || !CheckFinalTx(*wtx.tx) || wtx.GetBlocksToMaturity() > 0) { + continue; + } + + // Loop through tx outputs and add incoming payments. For outgoing txs, + // treat change outputs specially, as part of the amount debited. + CAmount debit = wtx.GetDebit(filter); + const bool outgoing = debit > 0; + for (const CTxOut& out : wtx.tx->vout) { + if (outgoing && IsChange(out)) { + debit -= out.nValue; + } else if (IsMine(out) & filter && depth >= minDepth && (!account || *account == GetAccountName(out.scriptPubKey))) { + balance += out.nValue; + } + } + + // For outgoing txs, subtract amount debited. + if (outgoing && (!account || *account == wtx.strFromAccount)) { + balance -= debit; + } + } + + if (account) { + balance += CWalletDB(*dbw).GetAccountCreditDebit(*account); + } + + return balance; +} + +CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const +{ + LOCK2(cs_main, cs_wallet); + + CAmount balance = 0; + std::vector<COutput> vCoins; + AvailableCoins(vCoins, true, coinControl); + for (const COutput& out : vCoins) { + if (out.fSpendable) { + balance += out.tx->tx->vout[out.i].nValue; + } + } + return balance; +} + +void CWallet::AvailableCoins(std::vector<COutput> &vCoins, bool fOnlySafe, const CCoinControl *coinControl, const CAmount &nMinimumAmount, const CAmount &nMaximumAmount, const CAmount &nMinimumSumAmount, const uint64_t &nMaximumCount, const int &nMinDepth, const int &nMaxDepth) const { vCoins.clear(); { LOCK2(cs_main, cs_wallet); + + CAmount nTotal = 0; + for (std::map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) { const uint256& wtxid = it->first; @@ -2047,21 +2061,115 @@ void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const continue; } + if (nDepth < nMinDepth || nDepth > nMaxDepth) + continue; + for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) { + if (pcoin->tx->vout[i].nValue < nMinimumAmount || pcoin->tx->vout[i].nValue > nMaximumAmount) + continue; + + if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint((*it).first, i))) + continue; + + if (IsLockedCoin((*it).first, i)) + continue; + + if (IsSpent(wtxid, i)) + continue; + isminetype mine = IsMine(pcoin->tx->vout[i]); - if (!(IsSpent(wtxid, i)) && mine != ISMINE_NO && - !IsLockedCoin((*it).first, i) && (pcoin->tx->vout[i].nValue > 0 || fIncludeZeroValue) && - (!coinControl || !coinControl->HasSelected() || coinControl->fAllowOtherInputs || coinControl->IsSelected(COutPoint((*it).first, i)))) - vCoins.push_back(COutput(pcoin, i, nDepth, - ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || - (coinControl && coinControl->fAllowWatchOnly && (mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO), - (mine & (ISMINE_SPENDABLE | ISMINE_WATCH_SOLVABLE)) != ISMINE_NO, safeTx)); + + if (mine == ISMINE_NO) { + continue; + } + + bool fSpendableIn = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (coinControl && coinControl->fAllowWatchOnly && (mine & ISMINE_WATCH_SOLVABLE) != ISMINE_NO); + bool fSolvableIn = (mine & (ISMINE_SPENDABLE | ISMINE_WATCH_SOLVABLE)) != ISMINE_NO; + + vCoins.push_back(COutput(pcoin, i, nDepth, fSpendableIn, fSolvableIn, safeTx)); + + // Checks the sum amount of all UTXO's. + if (nMinimumSumAmount != MAX_MONEY) { + nTotal += pcoin->tx->vout[i].nValue; + + if (nTotal >= nMinimumSumAmount) { + return; + } + } + + // Checks the maximum number of UTXO's. + if (nMaximumCount > 0 && vCoins.size() >= nMaximumCount) { + return; + } + } + } + } +} + +std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins() const +{ + // TODO: Add AssertLockHeld(cs_wallet) here. + // + // Because the return value from this function contains pointers to + // CWalletTx objects, callers to this function really should acquire the + // cs_wallet lock before calling it. However, the current caller doesn't + // acquire this lock yet. There was an attempt to add the missing lock in + // https://github.com/bitcoin/bitcoin/pull/10340, but that change has been + // postponed until after https://github.com/bitcoin/bitcoin/pull/10244 to + // avoid adding some extra complexity to the Qt code. + + std::map<CTxDestination, std::vector<COutput>> result; + + std::vector<COutput> availableCoins; + AvailableCoins(availableCoins); + + LOCK2(cs_main, cs_wallet); + for (auto& coin : availableCoins) { + CTxDestination address; + if (coin.fSpendable && + ExtractDestination(FindNonChangeParentOutput(*coin.tx->tx, coin.i).scriptPubKey, address)) { + result[address].emplace_back(std::move(coin)); + } + } + + std::vector<COutPoint> lockedCoins; + ListLockedCoins(lockedCoins); + for (const auto& output : lockedCoins) { + auto it = mapWallet.find(output.hash); + if (it != mapWallet.end()) { + int depth = it->second.GetDepthInMainChain(); + if (depth >= 0 && output.n < it->second.tx->vout.size() && + IsMine(it->second.tx->vout[output.n]) == ISMINE_SPENDABLE) { + CTxDestination address; + if (ExtractDestination(FindNonChangeParentOutput(*it->second.tx, output.n).scriptPubKey, address)) { + result[address].emplace_back( + &it->second, output.n, depth, true /* spendable */, true /* solvable */, false /* safe */); + } } } } + + return result; +} + +const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int output) const +{ + const CTransaction* ptx = &tx; + int n = output; + while (IsChange(ptx->vout[n]) && ptx->vin.size() > 0) { + const COutPoint& prevout = ptx->vin[0].prevout; + auto it = mapWallet.find(prevout.hash); + if (it == mapWallet.end() || it->second.tx->vout.size() <= prevout.n || + !IsMine(it->second.tx->vout[prevout.n])) { + break; + } + ptx = it->second.tx.get(); + n = prevout.n; + } + return ptx->vout[n]; } -static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair<const CWalletTx*,unsigned int> > >& vValue, const CAmount& nTotalLower, const CAmount& nTargetValue, +static void ApproximateBestSubset(const std::vector<CInputCoin>& vValue, const CAmount& nTotalLower, const CAmount& nTargetValue, std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000) { std::vector<char> vfIncluded; @@ -2086,9 +2194,9 @@ static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair //that the rng is fast. We do not use a constant random sequence, //because there may be some privacy improvement by making //the selection random. - if (nPass == 0 ? insecure_rand.rand32()&1 : !vfIncluded[i]) + if (nPass == 0 ? insecure_rand.randbool() : !vfIncluded[i]) { - nTotal += vValue[i].first; + nTotal += vValue[i].txout.nValue; vfIncluded[i] = true; if (nTotal >= nTargetValue) { @@ -2098,7 +2206,7 @@ static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair nBest = nTotal; vfBest = vfIncluded; } - nTotal -= vValue[i].first; + nTotal -= vValue[i].txout.nValue; vfIncluded[i] = false; } } @@ -2108,16 +2216,14 @@ static void ApproximateBestSubset(const std::vector<std::pair<CAmount, std::pair } bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMine, const int nConfTheirs, const uint64_t nMaxAncestors, std::vector<COutput> vCoins, - std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet) const + std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet) const { setCoinsRet.clear(); nValueRet = 0; // List of values less than target - std::pair<CAmount, std::pair<const CWalletTx*,unsigned int> > coinLowestLarger; - coinLowestLarger.first = std::numeric_limits<CAmount>::max(); - coinLowestLarger.second.first = NULL; - std::vector<std::pair<CAmount, std::pair<const CWalletTx*,unsigned int> > > vValue; + boost::optional<CInputCoin> coinLowestLarger; + std::vector<CInputCoin> vValue; CAmount nTotalLower = 0; random_shuffle(vCoins.begin(), vCoins.end(), GetRandInt); @@ -2136,22 +2242,21 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin continue; int i = output.i; - CAmount n = pcoin->tx->vout[i].nValue; - std::pair<CAmount,std::pair<const CWalletTx*,unsigned int> > coin = std::make_pair(n,std::make_pair(pcoin, i)); + CInputCoin coin = CInputCoin(pcoin, i); - if (n == nTargetValue) + if (coin.txout.nValue == nTargetValue) { - setCoinsRet.insert(coin.second); - nValueRet += coin.first; + setCoinsRet.insert(coin); + nValueRet += coin.txout.nValue; return true; } - else if (n < nTargetValue + MIN_CHANGE) + else if (coin.txout.nValue < nTargetValue + MIN_CHANGE) { vValue.push_back(coin); - nTotalLower += n; + nTotalLower += coin.txout.nValue; } - else if (n < coinLowestLarger.first) + else if (!coinLowestLarger || coin.txout.nValue < coinLowestLarger->txout.nValue) { coinLowestLarger = coin; } @@ -2159,20 +2264,20 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin if (nTotalLower == nTargetValue) { - for (unsigned int i = 0; i < vValue.size(); ++i) + for (const auto& input : vValue) { - setCoinsRet.insert(vValue[i].second); - nValueRet += vValue[i].first; + setCoinsRet.insert(input); + nValueRet += input.txout.nValue; } return true; } if (nTotalLower < nTargetValue) { - if (coinLowestLarger.second.first == NULL) + if (!coinLowestLarger) return false; - setCoinsRet.insert(coinLowestLarger.second); - nValueRet += coinLowestLarger.first; + setCoinsRet.insert(coinLowestLarger.get()); + nValueRet += coinLowestLarger->txout.nValue; return true; } @@ -2188,25 +2293,25 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin // If we have a bigger coin and (either the stochastic approximation didn't find a good solution, // or the next bigger coin is closer), return the bigger coin - if (coinLowestLarger.second.first && - ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || coinLowestLarger.first <= nBest)) + if (coinLowestLarger && + ((nBest != nTargetValue && nBest < nTargetValue + MIN_CHANGE) || coinLowestLarger->txout.nValue <= nBest)) { - setCoinsRet.insert(coinLowestLarger.second); - nValueRet += coinLowestLarger.first; + setCoinsRet.insert(coinLowestLarger.get()); + nValueRet += coinLowestLarger->txout.nValue; } else { for (unsigned int i = 0; i < vValue.size(); i++) if (vfBest[i]) { - setCoinsRet.insert(vValue[i].second); - nValueRet += vValue[i].first; + setCoinsRet.insert(vValue[i]); + nValueRet += vValue[i].txout.nValue; } if (LogAcceptCategory(BCLog::SELECTCOINS)) { LogPrint(BCLog::SELECTCOINS, "SelectCoins() best subset: "); for (unsigned int i = 0; i < vValue.size(); i++) { if (vfBest[i]) { - LogPrint(BCLog::SELECTCOINS, "%s ", FormatMoney(vValue[i].first)); + LogPrint(BCLog::SELECTCOINS, "%s ", FormatMoney(vValue[i].txout.nValue)); } } LogPrint(BCLog::SELECTCOINS, "total %s\n", FormatMoney(nBest)); @@ -2216,7 +2321,7 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin return true; } -bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet, const CCoinControl* coinControl) const +bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl* coinControl) const { std::vector<COutput> vCoins(vAvailableCoins); @@ -2228,13 +2333,13 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm if (!out.fSpendable) continue; nValueRet += out.tx->tx->vout[out.i].nValue; - setCoinsRet.insert(std::make_pair(out.tx, out.i)); + setCoinsRet.insert(CInputCoin(out.tx, out.i)); } return (nValueRet >= nTargetValue); } // calculate value from preset inputs and store them - std::set<std::pair<const CWalletTx*, uint32_t> > setPresetCoins; + std::set<CInputCoin> setPresetCoins; CAmount nValueFromPresetInputs = 0; std::vector<COutPoint> vPresetInputs; @@ -2250,7 +2355,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm if (pcoin->tx->vout.size() <= outpoint.n) return false; nValueFromPresetInputs += pcoin->tx->vout[outpoint.n].nValue; - setPresetCoins.insert(std::make_pair(pcoin, outpoint.n)); + setPresetCoins.insert(CInputCoin(pcoin, outpoint.n)); } else return false; // TODO: Allow non-wallet inputs } @@ -2258,7 +2363,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // remove preset inputs from vCoins for (std::vector<COutput>::iterator it = vCoins.begin(); it != vCoins.end() && coinControl && coinControl->HasSelected();) { - if (setPresetCoins.count(std::make_pair(it->tx, it->i))) + if (setPresetCoins.count(CInputCoin(it->tx, it->i))) it = vCoins.erase(it); else ++it; @@ -2287,10 +2392,12 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm bool CWallet::SignTransaction(CMutableTransaction &tx) { + AssertLockHeld(cs_wallet); // mapWallet + // sign the new tx CTransaction txNewConst(tx); int nIn = 0; - for (auto& input : tx.vin) { + for (const auto& input : tx.vin) { std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(input.prevout.hash); if(mi == mapWallet.end() || input.prevout.n >= mi->second.tx->vout.size()) { return false; @@ -2424,7 +2531,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT assert(txNew.nLockTime < LOCKTIME_THRESHOLD); { - std::set<std::pair<const CWalletTx*,unsigned int> > setCoins; + std::set<CInputCoin> setCoins; LOCK2(cs_main, cs_wallet); { std::vector<COutput> vAvailableCoins; @@ -2459,7 +2566,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT } } - if (txout.IsDust(dustRelayFee)) + if (IsDust(txout, ::dustRelayFee)) { if (recipient.fSubtractFeeFromAmount && nFeeRet > 0) { @@ -2524,16 +2631,16 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT // We do not move dust-change to fees, because the sender would end up paying more than requested. // This would be against the purpose of the all-inclusive feature. // So instead we raise the change and deduct from the recipient. - if (nSubtractFeeFromAmount > 0 && newTxOut.IsDust(dustRelayFee)) + if (nSubtractFeeFromAmount > 0 && IsDust(newTxOut, ::dustRelayFee)) { - CAmount nDust = newTxOut.GetDustThreshold(dustRelayFee) - newTxOut.nValue; + CAmount nDust = GetDustThreshold(newTxOut, ::dustRelayFee) - newTxOut.nValue; newTxOut.nValue += nDust; // raise change until no more dust for (unsigned int i = 0; i < vecSend.size(); i++) // subtract from first recipient { if (vecSend[i].fSubtractFeeFromAmount) { txNew.vout[i].nValue -= nDust; - if (txNew.vout[i].IsDust(dustRelayFee)) + if (IsDust(txNew.vout[i], ::dustRelayFee)) { strFailReason = _("The transaction amount is too small to send after the fee has been deducted"); return false; @@ -2545,7 +2652,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT // Never create dust outputs; if we would, just // add the dust to the fee. - if (newTxOut.IsDust(dustRelayFee)) + if (IsDust(newTxOut, ::dustRelayFee)) { nChangePosInOut = -1; nFeeRet += nChange; @@ -2567,9 +2674,10 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT std::vector<CTxOut>::iterator position = txNew.vout.begin()+nChangePosInOut; txNew.vout.insert(position, newTxOut); } - } - else + } else { reservekey.ReturnKey(); + nChangePosInOut = -1; + } // Fill vin // @@ -2583,7 +2691,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT // behavior." bool rbf = coinControl ? coinControl->signalRbf : fWalletRbf; for (const auto& coin : setCoins) - txNew.vin.push_back(CTxIn(coin.first->GetHash(),coin.second,CScript(), + txNew.vin.push_back(CTxIn(coin.outpoint,CScript(), std::numeric_limits<unsigned int>::max() - (rbf ? 2 : 1))); // Fill in dummy signatures for fee calculation. @@ -2607,10 +2715,7 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT if (coinControl && coinControl->nConfirmTarget > 0) currentConfirmationTarget = coinControl->nConfirmTarget; - CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, mempool); - if (coinControl && nFeeNeeded > 0 && coinControl->nMinimumTotalFee > nFeeNeeded) { - nFeeNeeded = coinControl->nMinimumTotalFee; - } + CAmount nFeeNeeded = GetMinimumFee(nBytes, currentConfirmationTarget, ::mempool, ::feeEstimator); if (coinControl && coinControl->fOverrideFeeRate) nFeeNeeded = coinControl->nFeeRate.GetFee(nBytes); @@ -2666,10 +2771,10 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT int nIn = 0; for (const auto& coin : setCoins) { - const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; + const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; - if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.first->tx->vout[coin.second].nValue, SIGHASH_ALL), scriptPubKey, sigdata)) + if (!ProduceSignature(TransactionSignatureCreator(this, &txNewConst, nIn, coin.txout.nValue, SIGHASH_ALL), scriptPubKey, sigdata)) { strFailReason = _("Signing transaction failed"); return false; @@ -2753,13 +2858,13 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon } void CWallet::ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& entries) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); return walletdb.ListAccountCreditDebit(strAccount, entries); } bool CWallet::AddAccountingEntry(const CAccountingEntry& acentry) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); return AddAccountingEntry(acentry, &walletdb); } @@ -2781,19 +2886,14 @@ CAmount CWallet::GetRequiredFee(unsigned int nTxBytes) return std::max(minTxFee.GetFee(nTxBytes), ::minRelayTxFee.GetFee(nTxBytes)); } -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool) +CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreGlobalPayTxFee) { // payTxFee is the user-set global for desired feerate - return GetMinimumFee(nTxBytes, nConfirmTarget, pool, payTxFee.GetFee(nTxBytes)); -} - -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, CAmount targetFee) -{ - CAmount nFeeNeeded = targetFee; + CAmount nFeeNeeded = payTxFee.GetFee(nTxBytes); // User didn't set: use -txconfirmtarget to estimate... - if (nFeeNeeded == 0) { + if (nFeeNeeded == 0 || ignoreGlobalPayTxFee) { int estimateFoundTarget = nConfirmTarget; - nFeeNeeded = pool.estimateSmartFee(nConfirmTarget, &estimateFoundTarget).GetFee(nTxBytes); + nFeeNeeded = estimator.estimateSmartFee(nConfirmTarget, &estimateFoundTarget, pool).GetFee(nTxBytes); // ... unless we don't have enough mempool data for estimatefee, then use fallbackFee if (nFeeNeeded == 0) nFeeNeeded = fallbackFee.GetFee(nTxBytes); @@ -2811,13 +2911,11 @@ CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarge DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { - if (!fFileBacked) - return DB_LOAD_OK; fFirstRunRet = false; - DBErrors nLoadWalletRet = CWalletDB(strWalletFile,"cr+").LoadWallet(this); + DBErrors nLoadWalletRet = CWalletDB(*dbw,"cr+").LoadWallet(this); if (nLoadWalletRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { LOCK(cs_wallet); setKeyPool.clear(); @@ -2838,17 +2936,15 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) { - if (!fFileBacked) - return DB_LOAD_OK; AssertLockHeld(cs_wallet); // mapWallet vchDefaultKey = CPubKey(); - DBErrors nZapSelectTxRet = CWalletDB(strWalletFile,"cr+").ZapSelectTx(vHashIn, vHashOut); + DBErrors nZapSelectTxRet = CWalletDB(*dbw,"cr+").ZapSelectTx(vHashIn, vHashOut); for (uint256 hash : vHashOut) mapWallet.erase(hash); if (nZapSelectTxRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { setKeyPool.clear(); // Note: can't top-up keypool here, because wallet is locked. @@ -2868,13 +2964,11 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) { - if (!fFileBacked) - return DB_LOAD_OK; vchDefaultKey = CPubKey(); - DBErrors nZapWalletTxRet = CWalletDB(strWalletFile,"cr+").ZapWalletTx(vWtx); + DBErrors nZapWalletTxRet = CWalletDB(*dbw,"cr+").ZapWalletTx(vWtx); if (nZapWalletTxRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { LOCK(cs_wallet); setKeyPool.clear(); @@ -2904,11 +2998,9 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s } NotifyAddressBookChanged(this, address, strName, ::IsMine(*this, address) != ISMINE_NO, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); - if (!fFileBacked) - return false; - if (!strPurpose.empty() && !CWalletDB(strWalletFile).WritePurpose(CBitcoinAddress(address).ToString(), strPurpose)) + if (!strPurpose.empty() && !CWalletDB(*dbw).WritePurpose(CBitcoinAddress(address).ToString(), strPurpose)) return false; - return CWalletDB(strWalletFile).WriteName(CBitcoinAddress(address).ToString(), strName); + return CWalletDB(*dbw).WriteName(CBitcoinAddress(address).ToString(), strName); } bool CWallet::DelAddressBook(const CTxDestination& address) @@ -2916,33 +3008,40 @@ bool CWallet::DelAddressBook(const CTxDestination& address) { LOCK(cs_wallet); // mapAddressBook - if(fFileBacked) + // Delete destdata tuples associated with address + std::string strAddress = CBitcoinAddress(address).ToString(); + BOOST_FOREACH(const PAIRTYPE(std::string, std::string) &item, mapAddressBook[address].destdata) { - // Delete destdata tuples associated with address - std::string strAddress = CBitcoinAddress(address).ToString(); - BOOST_FOREACH(const PAIRTYPE(std::string, std::string) &item, mapAddressBook[address].destdata) - { - CWalletDB(strWalletFile).EraseDestData(strAddress, item.first); - } + CWalletDB(*dbw).EraseDestData(strAddress, item.first); } mapAddressBook.erase(address); } NotifyAddressBookChanged(this, address, "", ::IsMine(*this, address) != ISMINE_NO, "", CT_DELETED); - if (!fFileBacked) - return false; - CWalletDB(strWalletFile).ErasePurpose(CBitcoinAddress(address).ToString()); - return CWalletDB(strWalletFile).EraseName(CBitcoinAddress(address).ToString()); + CWalletDB(*dbw).ErasePurpose(CBitcoinAddress(address).ToString()); + return CWalletDB(*dbw).EraseName(CBitcoinAddress(address).ToString()); } -bool CWallet::SetDefaultKey(const CPubKey &vchPubKey) +const std::string& CWallet::GetAccountName(const CScript& scriptPubKey) const { - if (fFileBacked) - { - if (!CWalletDB(strWalletFile).WriteDefaultKey(vchPubKey)) - return false; + CTxDestination address; + if (ExtractDestination(scriptPubKey, address) && !scriptPubKey.IsUnspendable()) { + auto mi = mapAddressBook.find(address); + if (mi != mapAddressBook.end()) { + return mi->second.name; + } } + // A scriptPubKey that doesn't have an entry in the address book is + // associated with the default account (""). + const static std::string DEFAULT_ACCOUNT_NAME; + return DEFAULT_ACCOUNT_NAME; +} + +bool CWallet::SetDefaultKey(const CPubKey &vchPubKey) +{ + if (!CWalletDB(*dbw).WriteDefaultKey(vchPubKey)) + return false; vchDefaultKey = vchPubKey; return true; } @@ -2955,7 +3054,7 @@ bool CWallet::NewKeyPool() { { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); BOOST_FOREACH(int64_t nIndex, setKeyPool) walletdb.ErasePool(nIndex); setKeyPool.clear(); @@ -2976,7 +3075,7 @@ size_t CWallet::KeypoolCountExternalKeys() if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT)) return setKeyPool.size(); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); // count amount of external keys size_t amountE = 0; @@ -3019,7 +3118,7 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) missingInternal = 0; } bool internal = false; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); for (int64_t i = missingInternal + missingExternal; i--;) { int64_t nEnd = 1; @@ -3050,7 +3149,7 @@ void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool int if(setKeyPool.empty()) return; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); // try to find a key that matches the internal/external filter for(const int64_t& id : setKeyPool) @@ -3076,11 +3175,8 @@ void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool int void CWallet::KeepKey(int64_t nIndex) { // Remove from key pool - if (fFileBacked) - { - CWalletDB walletdb(strWalletFile); - walletdb.ErasePool(nIndex); - } + CWalletDB walletdb(*dbw); + walletdb.ErasePool(nIndex); LogPrintf("keypool keep %d\n", nIndex); } @@ -3122,7 +3218,7 @@ int64_t CWallet::GetOldestKeyPoolTime() return GetTime(); CKeyPool keypool; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); if (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) { @@ -3161,9 +3257,9 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() { LOCK(cs_wallet); - BOOST_FOREACH(PAIRTYPE(uint256, CWalletTx) walletEntry, mapWallet) + for (const auto& walletEntry : mapWallet) { - CWalletTx *pcoin = &walletEntry.second; + const CWalletTx *pcoin = &walletEntry.second; if (!pcoin->IsTrusted()) continue; @@ -3201,9 +3297,9 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() std::set< std::set<CTxDestination> > groupings; std::set<CTxDestination> grouping; - BOOST_FOREACH(PAIRTYPE(uint256, CWalletTx) walletEntry, mapWallet) + for (const auto& walletEntry : mapWallet) { - CWalletTx *pcoin = &walletEntry.second; + const CWalletTx *pcoin = &walletEntry.second; if (pcoin->tx->vin.size() > 0) { @@ -3240,11 +3336,11 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() } // group lone addrs by themselves - for (unsigned int i = 0; i < pcoin->tx->vout.size(); i++) - if (IsMine(pcoin->tx->vout[i])) + for (const auto& txout : pcoin->tx->vout) + if (IsMine(txout)) { CTxDestination address; - if(!ExtractDestination(pcoin->tx->vout[i].scriptPubKey, address)) + if(!ExtractDestination(txout.scriptPubKey, address)) continue; grouping.insert(address); groupings.insert(grouping); @@ -3288,37 +3384,6 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() return ret; } -CAmount CWallet::GetAccountBalance(const std::string& strAccount, int nMinDepth, const isminefilter& filter) -{ - CWalletDB walletdb(strWalletFile); - return GetAccountBalance(walletdb, strAccount, nMinDepth, filter); -} - -CAmount CWallet::GetAccountBalance(CWalletDB& walletdb, const std::string& strAccount, int nMinDepth, const isminefilter& filter) -{ - CAmount nBalance = 0; - - // Tally wallet transactions - for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) - { - const CWalletTx& wtx = (*it).second; - if (!CheckFinalTx(wtx) || wtx.GetBlocksToMaturity() > 0 || wtx.GetDepthInMainChain() < 0) - continue; - - CAmount nReceived, nSent, nFee; - wtx.GetAccountAmounts(strAccount, nReceived, nSent, nFee, filter); - - if (nReceived != 0 && wtx.GetDepthInMainChain() >= nMinDepth) - nBalance += nReceived; - nBalance -= nSent + nFee; - } - - // Tally internal accounting entries - nBalance += walletdb.GetAccountCreditDebit(strAccount); - - return nBalance; -} - std::set<CTxDestination> CWallet::GetAccountAddresses(const std::string& strAccount) const { LOCK(cs_wallet); @@ -3370,7 +3435,7 @@ void CWallet::GetAllReserveKeys(std::set<CKeyID>& setAddress) const { setAddress.clear(); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); LOCK2(cs_main, cs_wallet); BOOST_FOREACH(const int64_t& id, setKeyPool) @@ -3386,17 +3451,6 @@ void CWallet::GetAllReserveKeys(std::set<CKeyID>& setAddress) const } } -void CWallet::UpdatedTransaction(const uint256 &hashTx) -{ - { - LOCK(cs_wallet); - // Only notify UI if this transaction is in this wallet - std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(hashTx); - if (mi != mapWallet.end()) - NotifyTransactionChanged(this, hashTx, CT_UPDATED); - } -} - void CWallet::GetScriptForMining(std::shared_ptr<CReserveScript> &script) { std::shared_ptr<CReserveKey> rKey = std::make_shared<CReserveKey>(this); @@ -3434,7 +3488,7 @@ bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const return (setLockedCoins.count(outpt) > 0); } -void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) +void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const { AssertLockHeld(cs_wallet); // setLockedCoins for (std::set<COutPoint>::iterator it = setLockedCoins.begin(); @@ -3603,18 +3657,14 @@ bool CWallet::AddDestData(const CTxDestination &dest, const std::string &key, co return false; mapAddressBook[dest].destdata.insert(std::make_pair(key, value)); - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteDestData(CBitcoinAddress(dest).ToString(), key, value); + return CWalletDB(*dbw).WriteDestData(CBitcoinAddress(dest).ToString(), key, value); } bool CWallet::EraseDestData(const CTxDestination &dest, const std::string &key) { if (!mapAddressBook[dest].destdata.erase(key)) return false; - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).EraseDestData(CBitcoinAddress(dest).ToString(), key); + return CWalletDB(*dbw).EraseDestData(CBitcoinAddress(dest).ToString(), key); } bool CWallet::LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value) @@ -3639,6 +3689,20 @@ bool CWallet::GetDestData(const CTxDestination &dest, const std::string &key, st return false; } +std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const +{ + LOCK(cs_wallet); + std::vector<std::string> values; + for (const auto& address : mapAddressBook) { + for (const auto& data : address.second.destdata) { + if (!data.first.compare(0, prefix.size(), prefix)) { + values.emplace_back(data.second); + } + } + } + return values; +} + std::string CWallet::GetWalletHelpString(bool showDebug) { std::string strUsage = HelpMessageGroup(_("Wallet options:")); @@ -3684,7 +3748,8 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) if (GetBoolArg("-zapwallettxes", false)) { uiInterface.InitMessage(_("Zapping all transactions from wallet...")); - CWallet *tempWallet = new CWallet(walletFile); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile)); + CWallet *tempWallet = new CWallet(std::move(dbw)); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DB_LOAD_OK) { InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); @@ -3699,7 +3764,8 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) int64_t nStart = GetTimeMillis(); bool fFirstRun = true; - CWallet *walletInstance = new CWallet(walletFile); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile)); + CWallet *walletInstance = new CWallet(std::move(dbw)); DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun); if (nLoadWalletRet != DB_LOAD_OK) { @@ -3790,7 +3856,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) CBlockIndex *pindexRescan = chainActive.Genesis(); if (!GetBoolArg("-rescan", false)) { - CWalletDB walletdb(walletFile); + CWalletDB walletdb(*walletInstance->dbw); CBlockLocator locator; if (walletdb.ReadBestBlock(locator)) pindexRescan = FindForkInGlobalIndex(chainActive, locator); @@ -3823,7 +3889,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) // Restore wallet transaction metadata after -zapwallettxes=1 if (GetBoolArg("-zapwallettxes", false) && GetArg("-zapwallettxes", "1") != "2") { - CWalletDB walletdb(walletFile); + CWalletDB walletdb(*walletInstance->dbw); BOOST_FOREACH(const CWalletTx& wtxOld, vWtx) { @@ -3867,7 +3933,7 @@ bool CWallet::InitLoadWallet() std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - if (walletFile.find_first_of("/\\") != std::string::npos) { + if (boost::filesystem::path(walletFile).filename() != walletFile) { return InitError(_("-wallet parameter must only specify a filename (not a path)")); } else if (SanitizeString(walletFile, SAFE_CHARS_FILENAME) != walletFile) { return InitError(_("Invalid characters in -wallet filename")); @@ -3983,38 +4049,7 @@ bool CWallet::ParameterInteraction() bool CWallet::BackupWallet(const std::string& strDest) { - if (!fFileBacked) - return false; - while (true) - { - { - LOCK(bitdb.cs_db); - if (!bitdb.mapFileUseCount.count(strWalletFile) || bitdb.mapFileUseCount[strWalletFile] == 0) - { - // Flush log data to the dat file - bitdb.CloseDb(strWalletFile); - bitdb.CheckpointLSN(strWalletFile); - bitdb.mapFileUseCount.erase(strWalletFile); - - // Copy wallet file - fs::path pathSrc = GetDataDir() / strWalletFile; - fs::path pathDest(strDest); - if (fs::is_directory(pathDest)) - pathDest /= strWalletFile; - - try { - fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); - LogPrintf("copied %s to %s\n", strWalletFile, pathDest.string()); - return true; - } catch (const fs::filesystem_error& e) { - LogPrintf("error copying %s to %s - %s\n", strWalletFile, pathDest.string(), e.what()); - return false; - } - } - } - MilliSleep(100); - } - return false; + return dbw->Backup(strDest); } CKeyPool::CKeyPool() diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index cf90d261a5..a3974bf00b 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -7,6 +7,7 @@ #define BITCOIN_WALLET_WALLET_H #include "amount.h" +#include "policy/feerate.h" #include "streams.h" #include "tinyformat.h" #include "ui_interface.h" @@ -73,6 +74,7 @@ class CReserveKey; class CScript; class CScheduler; class CTxMemPool; +class CBlockPolicyEstimator; class CWalletTx; /** (client) version numbers for particular wallet features */ @@ -450,9 +452,6 @@ public: void GetAmounts(std::list<COutputEntry>& listReceived, std::list<COutputEntry>& listSent, CAmount& nFee, std::string& strSentAccount, const isminefilter& filter) const; - void GetAccountAmounts(const std::string& strAccount, CAmount& nReceived, - CAmount& nSent, CAmount& nFee, const isminefilter& filter) const; - bool IsFromMe(const isminefilter& filter) const { return (GetDebit(filter) > 0); @@ -473,7 +472,34 @@ public: }; +class CInputCoin { +public: + CInputCoin(const CWalletTx* walletTx, unsigned int i) + { + if (!walletTx) + throw std::invalid_argument("walletTx should not be null"); + if (i >= walletTx->tx->vout.size()) + throw std::out_of_range("The output index is out of range"); + + outpoint = COutPoint(walletTx->GetHash(), i); + txout = walletTx->tx->vout[i]; + } + + COutPoint outpoint; + CTxOut txout; + + bool operator<(const CInputCoin& rhs) const { + return outpoint < rhs.outpoint; + } + + bool operator!=(const CInputCoin& rhs) const { + return outpoint != rhs.outpoint; + } + bool operator==(const CInputCoin& rhs) const { + return outpoint == rhs.outpoint; + } +}; class COutput { @@ -624,13 +650,15 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface { private: static std::atomic<bool> fFlushScheduled; + std::atomic<bool> fAbortRescan; + std::atomic<bool> fScanningWallet; /** * Select a set of coins such that nValueRet >= nTargetValue and at least * all coins from coinControl are selected; Never select unconfirmed coins * if they are not ours */ - bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet, const CCoinControl *coinControl = NULL) const; + bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl *coinControl = NULL) const; CWalletDB *pwalletdbEncryption; @@ -659,8 +687,9 @@ private: void SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator>); - /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected */ - void SyncTransaction(const CTransactionRef& tx, const CBlockIndex *pindexBlockConnected, int posInBlock); + /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected. + * Should be called with pindexBlock and posInBlock if this is for a transaction that is included in a block. */ + void SyncTransaction(const CTransactionRef& tx, const CBlockIndex *pindex = NULL, int posInBlock = 0); /* the HD chain data model (external chain counters) */ CHDChain hdChain; @@ -668,8 +697,6 @@ private: /* HD derive new child key (on internal or external chain) */ void DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret, bool internal = false); - bool fFileBacked; - std::set<int64_t> setKeyPool; int64_t nTimeFirstKey; @@ -685,17 +712,33 @@ private: */ bool AddWatchOnly(const CScript& dest) override; + std::unique_ptr<CWalletDBWrapper> dbw; + public: /* * Main wallet lock. - * This lock protects all the fields added by CWallet - * except for: - * fFileBacked (immutable after instantiation) - * strWalletFile (immutable after instantiation) + * This lock protects all the fields added by CWallet. */ mutable CCriticalSection cs_wallet; - const std::string strWalletFile; + /** Get database handle used by this wallet. Ideally this function would + * not be necessary. + */ + CWalletDBWrapper& GetDBHandle() + { + return *dbw; + } + + /** Get a name for this wallet for logging/debugging purposes. + */ + std::string GetName() const + { + if (dbw) { + return dbw->GetName(); + } else { + return "dummy"; + } + } void LoadKeyPool(int nIndex, const CKeyPool &keypool) { @@ -717,15 +760,16 @@ public: MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID; - CWallet() + // Create wallet with dummy database handle + CWallet(): dbw(new CWalletDBWrapper()) { SetNull(); } - CWallet(const std::string& strWalletFileIn) : strWalletFile(strWalletFileIn) + // Create wallet with passed-in database handle + CWallet(std::unique_ptr<CWalletDBWrapper> dbw_in) : dbw(std::move(dbw_in)) { SetNull(); - fFileBacked = true; } ~CWallet() @@ -738,7 +782,6 @@ public: { nWalletVersion = FEATURE_BASE; nWalletMaxVersion = FEATURE_BASE; - fFileBacked = false; nMasterKeyMaxID = 0; pwalletdbEncryption = NULL; nOrderPosNext = 0; @@ -747,6 +790,8 @@ public: nTimeFirstKey = 0; fBroadcastTransactions = false; nRelockTime = 0; + fAbortRescan = false; + fScanningWallet = false; } std::map<uint256, CWalletTx> mapWallet; @@ -773,7 +818,17 @@ public: /** * populate vCoins with vector of available COutputs. */ - void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = NULL, bool fIncludeZeroValue=false) const; + void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe=true, const CCoinControl *coinControl = NULL, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t& nMaximumCount = 0, const int& nMinDepth = 0, const int& nMaxDepth = 9999999) const; + + /** + * Return list of available coins and locked coins grouped by non-change output address. + */ + std::map<CTxDestination, std::vector<COutput>> ListCoins() const; + + /** + * Find non-change parent output. + */ + const CTxOut& FindNonChangeParentOutput(const CTransaction& tx, int output) const; /** * Shuffle and select coins until nTargetValue is reached while avoiding @@ -781,7 +836,7 @@ public: * completion the coin set and corresponding actual target value is * assembled */ - bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, uint64_t nMaxAncestors, std::vector<COutput> vCoins, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet) const; + bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, uint64_t nMaxAncestors, std::vector<COutput> vCoins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet) const; bool IsSpent(const uint256& hash, unsigned int n) const; @@ -789,7 +844,14 @@ public: void LockCoin(const COutPoint& output); void UnlockCoin(const COutPoint& output); void UnlockAllCoins(); - void ListLockedCoins(std::vector<COutPoint>& vOutpts); + void ListLockedCoins(std::vector<COutPoint>& vOutpts) const; + + /* + * Rescan abort properties + */ + void AbortRescan() { fAbortRescan = true; } + bool IsAbortingRescan() { return fAbortRescan; } + bool IsScanning() { return fScanningWallet; } /** * keystore implementation @@ -821,6 +883,8 @@ public: bool LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value); //! Look up a destination data tuple in the store, return true if found false otherwise bool GetDestData(const CTxDestination &dest, const std::string &key, std::string *value) const; + //! Get all destination values matching a prefix. + std::vector<std::string> GetDestValues(const std::string& prefix) const; //! Adds a watch-only address to the store, and saves it to disk. bool AddWatchOnly(const CScript& dest, int64_t nCreateTime); @@ -864,6 +928,8 @@ public: CAmount GetWatchOnlyBalance() const; CAmount GetUnconfirmedWatchOnlyBalance() const; CAmount GetImmatureWatchOnlyBalance() const; + CAmount GetLegacyBalance(const isminefilter& filter, int minDepth, const std::string* account) const; + CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; /** * Insert additional inputs into the transaction by @@ -893,12 +959,7 @@ public: * Estimate the minimum fee considering user set parameters * and the required fee */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool); - /** - * Estimate the minimum fee considering required fee and targetFee or if 0 - * then fee estimation for nConfirmTarget - */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, CAmount targetFee); + static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreGlobalPayTxFee = false); /** * Return the minimum required fee taking into account the * floating relay fee and user set minimum transaction fee @@ -918,8 +979,6 @@ public: std::set< std::set<CTxDestination> > GetAddressGroupings(); std::map<CTxDestination, CAmount> GetAddressBalances(); - CAmount GetAccountBalance(const std::string& strAccount, int nMinDepth, const isminefilter& filter); - CAmount GetAccountBalance(CWalletDB& walletdb, const std::string& strAccount, int nMinDepth, const isminefilter& filter); std::set<CTxDestination> GetAccountAddresses(const std::string& strAccount) const; isminetype IsMine(const CTxIn& txin) const; @@ -950,7 +1009,7 @@ public: bool DelAddressBook(const CTxDestination& address); - void UpdatedTransaction(const uint256 &hashTx) override; + const std::string& GetAccountName(const CScript& scriptPubKey) const; void Inventory(const uint256 &hash) override { @@ -1020,6 +1079,9 @@ public: /** Set whether this wallet broadcasts transactions. */ void SetBroadcastTransactions(bool broadcast) { fBroadcastTransactions = broadcast; } + /** Return whether transaction can be abandoned */ + bool TransactionCanBeAbandoned(const uint256& hashTx) const; + /* Mark a transaction (and it in-wallet descendants) as abandoned so its inputs may be respent. */ bool AbandonTransaction(const uint256& hashTx); @@ -1055,9 +1117,10 @@ public: CPubKey GenerateNewHDMasterKey(); /* Set the current HD master key (will reset the chain child index counters) - If possibleOldChain is provided, the parameters from the old chain (version) - will be preserved. */ - bool SetHDMasterKey(const CPubKey& key, CHDChain *possibleOldChain = nullptr); + Sets the master key's version based on the current wallet version (so the + caller must ensure the current wallet version is correct before calling + this function). */ + bool SetHDMasterKey(const CPubKey& key); }; /** A key allocated from the key pool. */ @@ -1130,7 +1193,7 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const ContainerType &coins int nIn = 0; for (const auto& coin : coins) { - const CScript& scriptPubKey = coin.first->tx->vout[coin.second].scriptPubKey; + const CScript& scriptPubKey = coin.txout.scriptPubKey; SignatureData sigdata; if (!ProduceSignature(DummySignatureCreator(this), scriptPubKey, sigdata)) diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index ceff2d36e3..342c797dd3 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -6,9 +6,9 @@ #include "wallet/walletdb.h" #include "base58.h" +#include "consensus/tx_verify.h" #include "consensus/validation.h" #include "fs.h" -#include "validation.h" // For CheckTransaction #include "protocol.h" #include "serialize.h" #include "sync.h" @@ -33,7 +33,7 @@ static std::atomic<unsigned int> nWalletDBUpdateCounter; bool CWalletDB::WriteName(const std::string& strAddress, const std::string& strName) { nWalletDBUpdateCounter++; - return Write(make_pair(std::string("name"), strAddress), strName); + return batch.Write(std::make_pair(std::string("name"), strAddress), strName); } bool CWalletDB::EraseName(const std::string& strAddress) @@ -41,38 +41,38 @@ bool CWalletDB::EraseName(const std::string& strAddress) // This should only be used for sending addresses, never for receiving addresses, // receiving addresses must always have an address book entry if they're not change return. nWalletDBUpdateCounter++; - return Erase(make_pair(std::string("name"), strAddress)); + return batch.Erase(std::make_pair(std::string("name"), strAddress)); } bool CWalletDB::WritePurpose(const std::string& strAddress, const std::string& strPurpose) { nWalletDBUpdateCounter++; - return Write(make_pair(std::string("purpose"), strAddress), strPurpose); + return batch.Write(std::make_pair(std::string("purpose"), strAddress), strPurpose); } bool CWalletDB::ErasePurpose(const std::string& strPurpose) { nWalletDBUpdateCounter++; - return Erase(make_pair(std::string("purpose"), strPurpose)); + return batch.Erase(std::make_pair(std::string("purpose"), strPurpose)); } bool CWalletDB::WriteTx(const CWalletTx& wtx) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("tx"), wtx.GetHash()), wtx); + return batch.Write(std::make_pair(std::string("tx"), wtx.GetHash()), wtx); } bool CWalletDB::EraseTx(uint256 hash) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("tx"), hash)); + return batch.Erase(std::make_pair(std::string("tx"), hash)); } bool CWalletDB::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, const CKeyMetadata& keyMeta) { nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("keymeta"), vchPubKey), + if (!batch.Write(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta, false)) return false; @@ -82,7 +82,7 @@ bool CWalletDB::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, c vchKey.insert(vchKey.end(), vchPubKey.begin(), vchPubKey.end()); vchKey.insert(vchKey.end(), vchPrivKey.begin(), vchPrivKey.end()); - return Write(std::make_pair(std::string("key"), vchPubKey), std::make_pair(vchPrivKey, Hash(vchKey.begin(), vchKey.end())), false); + return batch.Write(std::make_pair(std::string("key"), vchPubKey), std::make_pair(vchPrivKey, Hash(vchKey.begin(), vchKey.end())), false); } bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, @@ -92,16 +92,16 @@ bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, const bool fEraseUnencryptedKey = true; nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("keymeta"), vchPubKey), + if (!batch.Write(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta)) return false; - if (!Write(std::make_pair(std::string("ckey"), vchPubKey), vchCryptedSecret, false)) + if (!batch.Write(std::make_pair(std::string("ckey"), vchPubKey), vchCryptedSecret, false)) return false; if (fEraseUnencryptedKey) { - Erase(std::make_pair(std::string("key"), vchPubKey)); - Erase(std::make_pair(std::string("wkey"), vchPubKey)); + batch.Erase(std::make_pair(std::string("key"), vchPubKey)); + batch.Erase(std::make_pair(std::string("wkey"), vchPubKey)); } return true; } @@ -109,92 +109,92 @@ bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, bool CWalletDB::WriteMasterKey(unsigned int nID, const CMasterKey& kMasterKey) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("mkey"), nID), kMasterKey, true); + return batch.Write(std::make_pair(std::string("mkey"), nID), kMasterKey, true); } bool CWalletDB::WriteCScript(const uint160& hash, const CScript& redeemScript) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("cscript"), hash), *(const CScriptBase*)(&redeemScript), false); + return batch.Write(std::make_pair(std::string("cscript"), hash), *(const CScriptBase*)(&redeemScript), false); } bool CWalletDB::WriteWatchOnly(const CScript &dest, const CKeyMetadata& keyMeta) { nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) + if (!batch.Write(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) return false; - return Write(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); + return batch.Write(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); } bool CWalletDB::EraseWatchOnly(const CScript &dest) { nWalletDBUpdateCounter++; - if (!Erase(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) + if (!batch.Erase(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) return false; - return Erase(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); + return batch.Erase(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); } bool CWalletDB::WriteBestBlock(const CBlockLocator& locator) { nWalletDBUpdateCounter++; - Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan - return Write(std::string("bestblock_nomerkle"), locator); + batch.Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan + return batch.Write(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::ReadBestBlock(CBlockLocator& locator) { - if (Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; - return Read(std::string("bestblock_nomerkle"), locator); + if (batch.Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; + return batch.Read(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::WriteOrderPosNext(int64_t nOrderPosNext) { nWalletDBUpdateCounter++; - return Write(std::string("orderposnext"), nOrderPosNext); + return batch.Write(std::string("orderposnext"), nOrderPosNext); } bool CWalletDB::WriteDefaultKey(const CPubKey& vchPubKey) { nWalletDBUpdateCounter++; - return Write(std::string("defaultkey"), vchPubKey); + return batch.Write(std::string("defaultkey"), vchPubKey); } bool CWalletDB::ReadPool(int64_t nPool, CKeyPool& keypool) { - return Read(std::make_pair(std::string("pool"), nPool), keypool); + return batch.Read(std::make_pair(std::string("pool"), nPool), keypool); } bool CWalletDB::WritePool(int64_t nPool, const CKeyPool& keypool) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("pool"), nPool), keypool); + return batch.Write(std::make_pair(std::string("pool"), nPool), keypool); } bool CWalletDB::ErasePool(int64_t nPool) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("pool"), nPool)); + return batch.Erase(std::make_pair(std::string("pool"), nPool)); } bool CWalletDB::WriteMinVersion(int nVersion) { - return Write(std::string("minversion"), nVersion); + return batch.Write(std::string("minversion"), nVersion); } bool CWalletDB::ReadAccount(const std::string& strAccount, CAccount& account) { account.SetNull(); - return Read(make_pair(std::string("acc"), strAccount), account); + return batch.Read(std::make_pair(std::string("acc"), strAccount), account); } bool CWalletDB::WriteAccount(const std::string& strAccount, const CAccount& account) { - return Write(make_pair(std::string("acc"), strAccount), account); + return batch.Write(std::make_pair(std::string("acc"), strAccount), account); } bool CWalletDB::WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry) { - return Write(std::make_pair(std::string("acentry"), std::make_pair(acentry.strAccount, nAccEntryNum)), acentry); + return batch.Write(std::make_pair(std::string("acentry"), std::make_pair(acentry.strAccount, nAccEntryNum)), acentry); } bool CWalletDB::WriteAccountingEntry_Backend(const CAccountingEntry& acentry) @@ -218,7 +218,7 @@ void CWalletDB::ListAccountCreditDebit(const std::string& strAccount, std::list< { bool fAllAccounts = (strAccount == "*"); - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) throw std::runtime_error(std::string(__func__) + ": cannot create DB cursor"); bool setRange = true; @@ -229,7 +229,7 @@ void CWalletDB::ListAccountCreditDebit(const std::string& strAccount, std::list< if (setRange) ssKey << std::make_pair(std::string("acentry"), std::make_pair((fAllAccounts ? std::string("") : strAccount), uint64_t(0))); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue, setRange); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue, setRange); setRange = false; if (ret == DB_NOTFOUND) break; @@ -560,7 +560,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) LOCK(pwallet->cs_wallet); try { int nMinVersion = 0; - if (Read((std::string)"minversion", nMinVersion)) + if (batch.Read((std::string)"minversion", nMinVersion)) { if (nMinVersion > CLIENT_VERSION) return DB_TOO_NEW; @@ -568,7 +568,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) } // Get cursor - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) { LogPrintf("Error getting wallet database cursor\n"); @@ -580,7 +580,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue); if (ret == DB_NOTFOUND) break; else if (ret != 0) @@ -664,14 +664,14 @@ DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWal try { int nMinVersion = 0; - if (Read((std::string)"minversion", nMinVersion)) + if (batch.Read((std::string)"minversion", nMinVersion)) { if (nMinVersion > CLIENT_VERSION) return DB_TOO_NEW; } // Get cursor - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) { LogPrintf("Error getting wallet database cursor\n"); @@ -683,7 +683,7 @@ DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWal // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue); if (ret == DB_NOTFOUND) break; else if (ret != 0) @@ -797,9 +797,9 @@ void MaybeCompactWalletDB() if (nLastFlushed != CWalletDB::GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) { - const std::string& strFile = pwalletMain->strWalletFile; - if (CDB::PeriodicFlush(strFile)) + if (CDB::PeriodicFlush(pwalletMain->GetDBHandle())) { nLastFlushed = CWalletDB::GetUpdateCounter(); + } } fOneThread = false; } @@ -855,20 +855,20 @@ bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path bool CWalletDB::WriteDestData(const std::string &address, const std::string &key, const std::string &value) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("destdata"), std::make_pair(address, key)), value); + return batch.Write(std::make_pair(std::string("destdata"), std::make_pair(address, key)), value); } bool CWalletDB::EraseDestData(const std::string &address, const std::string &key) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("destdata"), std::make_pair(address, key))); + return batch.Erase(std::make_pair(std::string("destdata"), std::make_pair(address, key))); } bool CWalletDB::WriteHDChain(const CHDChain& chain) { nWalletDBUpdateCounter++; - return Write(std::string("hdchain"), chain); + return batch.Write(std::string("hdchain"), chain); } void CWalletDB::IncrementUpdateCounter() @@ -880,3 +880,28 @@ unsigned int CWalletDB::GetUpdateCounter() { return nWalletDBUpdateCounter; } + +bool CWalletDB::TxnBegin() +{ + return batch.TxnBegin(); +} + +bool CWalletDB::TxnCommit() +{ + return batch.TxnCommit(); +} + +bool CWalletDB::TxnAbort() +{ + return batch.TxnAbort(); +} + +bool CWalletDB::ReadVersion(int& nVersion) +{ + return batch.ReadVersion(nVersion); +} + +bool CWalletDB::WriteVersion(int nVersion) +{ + return batch.WriteVersion(nVersion); +} diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index b94f341b2e..cd9fe279c5 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -17,6 +17,21 @@ #include <utility> #include <vector> +/** + * Overview of wallet database classes: + * + * - CDBEnv is an environment in which the database exists (has no analog in dbwrapper.h) + * - CWalletDBWrapper represents a wallet database (similar to CDBWrapper in dbwrapper.h) + * - CDB is a low-level database transaction (similar to CDBBatch in dbwrapper.h) + * - CWalletDB is a modifier object for the wallet, and encapsulates a database + * transaction as well as methods to act on the database (no analog in + * dbwrapper.h) + * + * The latter two are named confusingly, in contrast to what the names CDB + * and CWalletDB suggest they are transient transaction objects and don't + * represent the database itself. + */ + static const bool DEFAULT_FLUSHWALLET = true; class CAccount; @@ -118,11 +133,16 @@ public: } }; -/** Access to the wallet database */ -class CWalletDB : public CDB +/** Access to the wallet database. + * This should really be named CWalletDBBatch, as it represents a single transaction at the + * database. It will be committed when the object goes out of scope. + * Optionally (on by default) it will flush to disk as well. + */ +class CWalletDB { public: - CWalletDB(const std::string& strFilename, const char* pszMode = "r+", bool _fFlushOnClose = true) : CDB(strFilename, pszMode, _fFlushOnClose) + CWalletDB(CWalletDBWrapper& dbw, const char* pszMode = "r+", bool _fFlushOnClose = true) : + batch(dbw, pszMode, _fFlushOnClose) { } @@ -194,7 +214,20 @@ public: static void IncrementUpdateCounter(); static unsigned int GetUpdateCounter(); + + //! Begin a new transaction + bool TxnBegin(); + //! Commit current transaction + bool TxnCommit(); + //! Abort current transaction + bool TxnAbort(); + //! Read wallet version + bool ReadVersion(int& nVersion); + //! Write wallet version + bool WriteVersion(int nVersion); private: + CDB batch; + CWalletDB(const CWalletDB&); void operator=(const CWalletDB&); }; diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index b2963e9bde..700c39f66e 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "chain.h" #include "chainparams.h" #include "streams.h" #include "zmqpublishnotifier.h" diff --git a/test/README.md b/test/README.md index dec8db960d..4dd512638d 100644 --- a/test/README.md +++ b/test/README.md @@ -25,6 +25,8 @@ The ZMQ functional test requires a python ZMQ library. To install it: Running tests locally ===================== +Build for your system first. Be sure to enable wallet, utils and daemon when you configure. Tests will not run otherwise. + Functional tests ---------------- @@ -32,7 +34,7 @@ You can run any single test by calling test/functional/test_runner.py <testname> -Or you can run any combination of tests by calling +Or you can run any combination (incl. duplicates) of tests by calling test/functional/test_runner.py <testname1> <testname2> <testname3> ... diff --git a/test/functional/README.md b/test/functional/README.md index 651b01f18a..e6c4849702 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -59,7 +59,7 @@ thread.) * RPC calls are available in p2p tests. * Can be used to write free-form tests, where specific p2p-protocol behavior -is tested. Examples: ```p2p-accept-block.py```, ```maxblocksinflight.py```. +is tested. Examples: ```p2p-accept-block.py```, ```p2p-compactblocks.py```. ## Comptool diff --git a/test/functional/abandonconflict.py b/test/functional/abandonconflict.py index 887dbebd4f..c87c02492d 100755 --- a/test/functional/abandonconflict.py +++ b/test/functional/abandonconflict.py @@ -10,22 +10,15 @@ which are not included in a block and are not currently in the mempool. It has no effect on transactions which are already conflicted or abandoned. """ - from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -import urllib.parse class AbandonConflictTest(BitcoinTestFramework): def __init__(self): super().__init__() self.num_nodes = 2 self.setup_clean_chain = False - - def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.00001"])) - self.nodes.append(start_node(1, self.options.tmpdir)) - connect_nodes(self.nodes[0], 1) + self.extra_args = [["-minrelaytxfee=0.00001"], []] def run_test(self): self.nodes[1].generate(100) @@ -42,8 +35,8 @@ class AbandonConflictTest(BitcoinTestFramework): assert(balance - newbalance < Decimal("0.001")) #no more than fees lost balance = newbalance - url = urllib.parse.urlparse(self.nodes[1].url) - self.nodes[0].disconnectnode(url.hostname+":"+str(p2p_port(1))) + # Disconnect nodes so node0's transactions don't get into node1's mempool + disconnect_nodes(self.nodes[0], 1) # Identify the 10btc outputs nA = next(i for i, vout in enumerate(self.nodes[0].getrawtransaction(txA, 1)["vout"]) if vout["value"] == Decimal("10")) @@ -80,11 +73,12 @@ class AbandonConflictTest(BitcoinTestFramework): # Restart the node with a higher min relay fee so the parent tx is no longer in mempool # TODO: redo with eviction - stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.0001"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.0001"]) - # Verify txs no longer in mempool + # Verify txs no longer in either node's mempool assert_equal(len(self.nodes[0].getrawmempool()), 0) + assert_equal(len(self.nodes[1].getrawmempool()), 0) # Not in mempool txs from self should only reduce balance # inputs are still spent, but change not received @@ -106,8 +100,8 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Verify that even with a low min relay fee, the tx is not reaccepted from wallet on startup once abandoned - stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.00001"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.00001"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) assert_equal(self.nodes[0].getbalance(), balance) @@ -126,8 +120,8 @@ class AbandonConflictTest(BitcoinTestFramework): balance = newbalance # Remove using high relay fee again - stop_node(self.nodes[0],0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.0001"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-minrelaytxfee=0.0001"]) assert_equal(len(self.nodes[0].getrawmempool()), 0) newbalance = self.nodes[0].getbalance() assert_equal(newbalance, balance - Decimal("24.9996")) diff --git a/test/functional/assumevalid.py b/test/functional/assumevalid.py index 8e301c4379..9d17faac51 100755 --- a/test/functional/assumevalid.py +++ b/test/functional/assumevalid.py @@ -45,7 +45,7 @@ from test_framework.mininode import (CBlockHeader, msg_headers) from test_framework.script import (CScript, OP_TRUE) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (start_node, p2p_port, assert_equal) +from test_framework.util import (p2p_port, assert_equal) class BaseNode(NodeConnCB): def send_header_for_blocks(self, new_blocks): @@ -63,7 +63,7 @@ class AssumeValidTest(BitcoinTestFramework): # Start node0. We don't start the other nodes yet since # we need to pre-mine a block with an invalid transaction # signature so we can pass in the block hash as assumevalid. - self.nodes = [start_node(0, self.options.tmpdir)] + self.nodes = [self.start_node(0, self.options.tmpdir)] def send_blocks_until_disconnected(self, node): """Keep sending blocks to the node until we're disconnected.""" @@ -162,14 +162,14 @@ class AssumeValidTest(BitcoinTestFramework): height += 1 # Start node1 and node2 with assumevalid so they accept a block with a bad signature. - self.nodes.append(start_node(1, self.options.tmpdir, + self.nodes.append(self.start_node(1, self.options.tmpdir, ["-assumevalid=" + hex(block102.sha256)])) node1 = BaseNode() # connects to node1 connections.append(NodeConn('127.0.0.1', p2p_port(1), self.nodes[1], node1)) node1.add_connection(connections[1]) node1.wait_for_verack() - self.nodes.append(start_node(2, self.options.tmpdir, + self.nodes.append(self.start_node(2, self.options.tmpdir, ["-assumevalid=" + hex(block102.sha256)])) node2 = BaseNode() # connects to node2 connections.append(NodeConn('127.0.0.1', p2p_port(2), self.nodes[2], node2)) diff --git a/test/functional/bip65-cltv-p2p.py b/test/functional/bip65-cltv-p2p.py index 63d05e8fc9..bb83042f35 100755 --- a/test/functional/bip65-cltv-p2p.py +++ b/test/functional/bip65-cltv-p2p.py @@ -39,12 +39,7 @@ class BIP65Test(ComparisonTestFramework): def __init__(self): super().__init__() self.num_nodes = 1 - - def setup_network(self): - # Must set the blockversion for this test - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-whitelist=127.0.0.1', '-blockversion=3']], - binary=[self.options.testbinary]) + self.extra_args = [['-whitelist=127.0.0.1', '-blockversion=3']] def run_test(self): test = TestManager(self, self.options.tmpdir) diff --git a/test/functional/bip65-cltv.py b/test/functional/bip65-cltv.py index 7f13bb9952..ddf932c746 100755 --- a/test/functional/bip65-cltv.py +++ b/test/functional/bip65-cltv.py @@ -12,15 +12,12 @@ class BIP65Test(BitcoinTestFramework): super().__init__() self.num_nodes = 3 self.setup_clean_chain = False + self.extra_args = [[], ["-blockversion=3"], ["-blockversion=4"]] def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, [])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-blockversion=3"])) - self.nodes.append(start_node(2, self.options.tmpdir, ["-blockversion=4"])) + self.setup_nodes() connect_nodes(self.nodes[1], 0) connect_nodes(self.nodes[2], 0) - self.is_network_split = False self.sync_all() def run_test(self): diff --git a/test/functional/bip68-112-113-p2p.py b/test/functional/bip68-112-113-p2p.py index 0867f42585..5a322e8c0e 100755 --- a/test/functional/bip68-112-113-p2p.py +++ b/test/functional/bip68-112-113-p2p.py @@ -95,12 +95,7 @@ class BIP68_112_113Test(ComparisonTestFramework): def __init__(self): super().__init__() self.num_nodes = 1 - - def setup_network(self): - # Must set the blockversion for this test - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-whitelist=127.0.0.1', '-blockversion=4']], - binary=[self.options.testbinary]) + self.extra_args = [['-whitelist=127.0.0.1', '-blockversion=4']] def run_test(self): test = TestManager(self, self.options.tmpdir) diff --git a/test/functional/bip68-sequence.py b/test/functional/bip68-sequence.py index 89d42a710c..db66b7719c 100755 --- a/test/functional/bip68-sequence.py +++ b/test/functional/bip68-sequence.py @@ -21,16 +21,11 @@ class BIP68Test(BitcoinTestFramework): super().__init__() self.num_nodes = 2 self.setup_clean_chain = False + self.extra_args = [[], ["-acceptnonstdtxn=0"]] - def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir)) - self.nodes.append(start_node(1, self.options.tmpdir, ["-acceptnonstdtxn=0"])) - self.is_network_split = False + def run_test(self): self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] - connect_nodes(self.nodes[0], 1) - def run_test(self): # Generate some coins self.nodes[0].generate(110) diff --git a/test/functional/bip9-softforks.py b/test/functional/bip9-softforks.py index 60d262da81..b90b0ca628 100755 --- a/test/functional/bip9-softforks.py +++ b/test/functional/bip9-softforks.py @@ -16,7 +16,6 @@ test that enforcement has not triggered (which triggers ACTIVE) test that enforcement has triggered """ -from test_framework.blockstore import BlockStore from test_framework.test_framework import ComparisonTestFramework from test_framework.util import * from test_framework.mininode import CTransaction, NetworkThread @@ -32,11 +31,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): def __init__(self): super().__init__() self.num_nodes = 1 - - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-whitelist=127.0.0.1']], - binary=[self.options.testbinary]) + self.extra_args = [['-whitelist=127.0.0.1']] def run_test(self): self.test = TestManager(self, self.options.tmpdir) @@ -104,12 +99,43 @@ class BIP9SoftForksTest(ComparisonTestFramework): assert_equal(self.get_bip9_status(bipName)['status'], 'started') assert_equal(self.get_bip9_status(bipName)['since'], 144) + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 0) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) assert_equal(tmpl['vbavailable'][bipName], bitno) assert_equal(tmpl['vbrequired'], 0) assert(tmpl['version'] & activated_version) + # Test 1-A + # check stats after max number of "signalling not" blocks such that LOCKED_IN still possible this period + test_blocks = self.generate_blocks(36, 4, test_blocks) # 0x00000004 (signalling not) + test_blocks = self.generate_blocks(10, activated_version) # 0x20000001 (signalling ready) + yield TestInstance(test_blocks, sync_every_block=False) + + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 46) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 10) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], True) + + # Test 1-B + # check stats after one additional "signalling not" block -- LOCKED_IN no longer possible this period + test_blocks = self.generate_blocks(1, 4, test_blocks) # 0x00000004 (signalling not) + yield TestInstance(test_blocks, sync_every_block=False) + + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 47) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 10) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], False) + + # Test 1-C + # finish period with "ready" blocks, but soft fork will still fail to advance to LOCKED_IN + test_blocks = self.generate_blocks(97, activated_version) # 0x20000001 (signalling ready) + yield TestInstance(test_blocks, sync_every_block=False) + + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], True) + assert_equal(self.get_bip9_status(bipName)['status'], 'started') + # Test 2 # Fail to achieve LOCKED_IN 100 out of 144 signal bit 1 # using a variety of bits to simulate multiple parallel softforks @@ -121,6 +147,8 @@ class BIP9SoftForksTest(ComparisonTestFramework): assert_equal(self.get_bip9_status(bipName)['status'], 'started') assert_equal(self.get_bip9_status(bipName)['since'], 144) + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 0) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 0) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) assert_equal(tmpl['vbavailable'][bipName], bitno) @@ -130,14 +158,24 @@ class BIP9SoftForksTest(ComparisonTestFramework): # Test 3 # 108 out of 144 signal bit 1 to achieve LOCKED_IN # using a variety of bits to simulate multiple parallel softforks - test_blocks = self.generate_blocks(58, activated_version) # 0x20000001 (signalling ready) + test_blocks = self.generate_blocks(57, activated_version) # 0x20000001 (signalling ready) test_blocks = self.generate_blocks(26, 4, test_blocks) # 0x00000004 (signalling not) test_blocks = self.generate_blocks(50, activated_version, test_blocks) # 0x20000101 (signalling ready) test_blocks = self.generate_blocks(10, 4, test_blocks) # 0x20010000 (signalling not) yield TestInstance(test_blocks, sync_every_block=False) + # check counting stats and "possible" flag before last block of this period achieves LOCKED_IN... + assert_equal(self.get_bip9_status(bipName)['statistics']['elapsed'], 143) + assert_equal(self.get_bip9_status(bipName)['statistics']['count'], 107) + assert_equal(self.get_bip9_status(bipName)['statistics']['possible'], True) + assert_equal(self.get_bip9_status(bipName)['status'], 'started') + + # ...continue with Test 3 + test_blocks = self.generate_blocks(1, activated_version) # 0x20000001 (signalling ready) + yield TestInstance(test_blocks, sync_every_block=False) + assert_equal(self.get_bip9_status(bipName)['status'], 'locked_in') - assert_equal(self.get_bip9_status(bipName)['since'], 432) + assert_equal(self.get_bip9_status(bipName)['since'], 576) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) @@ -147,7 +185,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): yield TestInstance(test_blocks, sync_every_block=False) assert_equal(self.get_bip9_status(bipName)['status'], 'locked_in') - assert_equal(self.get_bip9_status(bipName)['since'], 432) + assert_equal(self.get_bip9_status(bipName)['since'], 576) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName not in tmpl['rules']) @@ -173,7 +211,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): yield TestInstance([[block, True]]) assert_equal(self.get_bip9_status(bipName)['status'], 'active') - assert_equal(self.get_bip9_status(bipName)['since'], 576) + assert_equal(self.get_bip9_status(bipName)['since'], 720) tmpl = self.nodes[0].getblocktemplate({}) assert(bipName in tmpl['rules']) assert(bipName not in tmpl['vbavailable']) @@ -201,7 +239,7 @@ class BIP9SoftForksTest(ComparisonTestFramework): # Restart all self.test.clear_all_connections() - stop_nodes(self.nodes) + self.stop_nodes() shutil.rmtree(self.options.tmpdir + "/node0") self.setup_chain() self.setup_network() diff --git a/test/functional/bipdersig-p2p.py b/test/functional/bipdersig-p2p.py index 22bd39fbe5..31c7ebba90 100755 --- a/test/functional/bipdersig-p2p.py +++ b/test/functional/bipdersig-p2p.py @@ -47,12 +47,6 @@ class BIP66Test(ComparisonTestFramework): super().__init__() self.num_nodes = 1 - def setup_network(self): - # Must set the blockversion for this test - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-whitelist=127.0.0.1', '-blockversion=2']], - binary=[self.options.testbinary]) - def run_test(self): test = TestManager(self, self.options.tmpdir) test.add_all_connections(self.nodes) diff --git a/test/functional/bipdersig.py b/test/functional/bipdersig.py index 371cc41bb7..41f88fb664 100755 --- a/test/functional/bipdersig.py +++ b/test/functional/bipdersig.py @@ -12,15 +12,12 @@ class BIP66Test(BitcoinTestFramework): super().__init__() self.num_nodes = 3 self.setup_clean_chain = False + self.extra_args = [[], ["-blockversion=2"], ["-blockversion=3"]] def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, [])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-blockversion=2"])) - self.nodes.append(start_node(2, self.options.tmpdir, ["-blockversion=3"])) + self.setup_nodes() connect_nodes(self.nodes[1], 0) connect_nodes(self.nodes[2], 0) - self.is_network_split = False self.sync_all() def run_test(self): diff --git a/test/functional/blockchain.py b/test/functional/blockchain.py index 596aed50ec..5ada3e629e 100755 --- a/test/functional/blockchain.py +++ b/test/functional/blockchain.py @@ -6,6 +6,12 @@ Test the following RPCs: - gettxoutsetinfo + - getdifficulty + - getbestblockhash + - getblockhash + - getblockheader + - getchaintxstats + - getnetworkhashps - verifychain Tests correspond to code in rpc/blockchain.cpp. @@ -19,8 +25,6 @@ from test_framework.util import ( assert_raises_jsonrpc, assert_is_hex_string, assert_is_hash_string, - start_nodes, - connect_nodes_bi, ) @@ -29,19 +33,24 @@ class BlockchainTest(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = False - self.num_nodes = 2 - - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - connect_nodes_bi(self.nodes, 0, 1) - self.is_network_split = False - self.sync_all() + self.num_nodes = 1 def run_test(self): + self._test_getchaintxstats() self._test_gettxoutsetinfo() self._test_getblockheader() + self._test_getdifficulty() + self._test_getnetworkhashps() self.nodes[0].verifychain(4, 0) + def _test_getchaintxstats(self): + chaintxstats = self.nodes[0].getchaintxstats(1) + # 200 txs plus genesis tx + assert_equal(chaintxstats['txcount'], 201) + # tx rate should be 1 per 10 minutes, or 1/600 + # we have to round because of binary math + assert_equal(round(chaintxstats['txrate'] * 600, 10), Decimal(1)) + def _test_gettxoutsetinfo(self): node = self.nodes[0] res = node.gettxoutsetinfo() @@ -50,14 +59,41 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(res['transactions'], 200) assert_equal(res['height'], 200) assert_equal(res['txouts'], 200) - assert_equal(res['bytes_serialized'], 13924), + assert_equal(res['bestblock'], node.getblockhash(200)) + size = res['disk_size'] + assert size > 6400 + assert size < 64000 assert_equal(len(res['bestblock']), 64) - assert_equal(len(res['hash_serialized']), 64) + assert_equal(len(res['hash_serialized_2']), 64) + + self.log.info("Test that gettxoutsetinfo() works for blockchain with just the genesis block") + b1hash = node.getblockhash(1) + node.invalidateblock(b1hash) + + res2 = node.gettxoutsetinfo() + assert_equal(res2['transactions'], 0) + assert_equal(res2['total_amount'], Decimal('0')) + assert_equal(res2['height'], 0) + assert_equal(res2['txouts'], 0) + assert_equal(res2['bestblock'], node.getblockhash(0)) + assert_equal(len(res2['hash_serialized_2']), 64) + + self.log.info("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block") + node.reconsiderblock(b1hash) + + res3 = node.gettxoutsetinfo() + assert_equal(res['total_amount'], res3['total_amount']) + assert_equal(res['transactions'], res3['transactions']) + assert_equal(res['height'], res3['height']) + assert_equal(res['txouts'], res3['txouts']) + assert_equal(res['bestblock'], res3['bestblock']) + assert_equal(res['hash_serialized_2'], res3['hash_serialized_2']) def _test_getblockheader(self): node = self.nodes[0] - assert_raises_jsonrpc(-5, "Block not found", node.getblockheader, "nonsense") + assert_raises_jsonrpc(-5, "Block not found", + node.getblockheader, "nonsense") besthash = node.getbestblockhash() secondbesthash = node.getblockhash(199) @@ -79,5 +115,16 @@ class BlockchainTest(BitcoinTestFramework): assert isinstance(int(header['versionHex'], 16), int) assert isinstance(header['difficulty'], Decimal) + def _test_getdifficulty(self): + difficulty = self.nodes[0].getdifficulty() + # 1 hash in 2 should be valid, so difficulty should be 1/2**31 + # binary => decimal => binary math is why we do this check + assert abs(difficulty * 2**31 - 1) < 0.0001 + + def _test_getnetworkhashps(self): + hashes_per_second = self.nodes[0].getnetworkhashps() + # This should be 2 hashes every 10 minutes or 1/300 + assert abs(hashes_per_second * 300 - 1) < 0.0001 + if __name__ == '__main__': BlockchainTest().main() diff --git a/test/functional/bumpfee.py b/test/functional/bumpfee.py index c51a75cc26..d42bab6cbf 100755 --- a/test/functional/bumpfee.py +++ b/test/functional/bumpfee.py @@ -38,16 +38,15 @@ class BumpFeeTest(BitcoinTestFramework): def setup_network(self, split=False): extra_args = [["-prematurewitness", "-walletprematurewitness", "-walletrbf={}".format(i)] for i in range(self.num_nodes)] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) # Encrypt wallet for test_locked_wallet_fails test self.nodes[1].encryptwallet(WALLET_PASSPHRASE) bitcoind_processes[1].wait() - self.nodes[1] = start_node(1, self.options.tmpdir, extra_args[1]) + self.nodes[1] = self.start_node(1, self.options.tmpdir, extra_args[1]) self.nodes[1].walletpassphrase(WALLET_PASSPHRASE, WALLET_PASSPHRASE_TIMEOUT) connect_nodes_bi(self.nodes, 0, 1) - self.is_network_split = False self.sync_all() def run_test(self): @@ -89,6 +88,7 @@ def test_simple_bumpfee_succeeds(rbf_node, peer_node, dest_address): sync_mempools((rbf_node, peer_node)) assert rbfid in rbf_node.getrawmempool() and rbfid in peer_node.getrawmempool() bumped_tx = rbf_node.bumpfee(rbfid) + assert_equal(bumped_tx["errors"], []) assert bumped_tx["fee"] - abs(rbftx["fee"]) > 0 # check that bumped_tx propogates, original tx was evicted and has a wallet conflict sync_mempools((rbf_node, peer_node)) diff --git a/test/functional/decodescript.py b/test/functional/decodescript.py index 5555e96c44..21a9f1223f 100755 --- a/test/functional/decodescript.py +++ b/test/functional/decodescript.py @@ -16,10 +16,6 @@ class DecodeScriptTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - self.is_network_split = False - def decodescript_script_sig(self): signature = '304502207fa7a6d1e0ee81132a269ad84e68d695483745cde8b541e3bf630749894e342a022100c1f7ab20e13e22fb95281a870f3dcf38d782e53023ee313d741ad0cfbc0c509001' push_signature = '48' + signature diff --git a/test/functional/disablewallet.py b/test/functional/disablewallet.py index 2f729e19bf..d344513414 100755 --- a/test/functional/disablewallet.py +++ b/test/functional/disablewallet.py @@ -18,13 +18,11 @@ class DisableWalletTest (BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 1 - - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [['-disablewallet']]) - self.is_network_split = False - self.sync_all() + self.extra_args = [["-disablewallet"]] def run_test (self): + # Make sure wallet is really disabled + assert_raises_jsonrpc(-32601, 'Method not found', self.nodes[0].getwalletinfo) x = self.nodes[0].validateaddress('3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') assert(x['isvalid'] == False) x = self.nodes[0].validateaddress('mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') diff --git a/test/functional/disconnect_ban.py b/test/functional/disconnect_ban.py new file mode 100755 index 0000000000..89b68aeb25 --- /dev/null +++ b/test/functional/disconnect_ban.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test node disconnect and ban behavior""" +import time + +from test_framework.mininode import wait_until +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import (assert_equal, + assert_raises_jsonrpc, + connect_nodes_bi) + +class DisconnectBanTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.num_nodes = 2 + self.setup_clean_chain = False + + def run_test(self): + self.log.info("Test setban and listbanned RPCs") + + self.log.info("setban: successfully ban single IP address") + assert_equal(len(self.nodes[1].getpeerinfo()), 2) # node1 should have 2 connections to node0 at this point + self.nodes[1].setban("127.0.0.1", "add") + assert wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) + assert_equal(len(self.nodes[1].getpeerinfo()), 0) # all nodes must be disconnected at this point + assert_equal(len(self.nodes[1].listbanned()), 1) + + self.log.info("clearbanned: successfully clear ban list") + self.nodes[1].clearbanned() + assert_equal(len(self.nodes[1].listbanned()), 0) + self.nodes[1].setban("127.0.0.0/24", "add") + + self.log.info("setban: fail to ban an already banned subnet") + assert_equal(len(self.nodes[1].listbanned()), 1) + assert_raises_jsonrpc(-23, "IP/Subnet already banned", self.nodes[1].setban, "127.0.0.1", "add") + + self.log.info("setban: fail to ban an invalid subnet") + assert_raises_jsonrpc(-30, "Error: Invalid IP/Subnet", self.nodes[1].setban, "127.0.0.1/42", "add") + assert_equal(len(self.nodes[1].listbanned()), 1) # still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 + + self.log.info("setban remove: fail to unban a non-banned subnet") + assert_raises_jsonrpc(-30, "Error: Unban failed", self.nodes[1].setban, "127.0.0.1", "remove") + assert_equal(len(self.nodes[1].listbanned()), 1) + + self.log.info("setban remove: successfully unban subnet") + self.nodes[1].setban("127.0.0.0/24", "remove") + assert_equal(len(self.nodes[1].listbanned()), 0) + self.nodes[1].clearbanned() + assert_equal(len(self.nodes[1].listbanned()), 0) + + self.log.info("setban: test persistence across node restart") + self.nodes[1].setban("127.0.0.0/32", "add") + self.nodes[1].setban("127.0.0.0/24", "add") + # Set the mocktime so we can control when bans expire + old_time = int(time.time()) + self.nodes[1].setmocktime(old_time) + self.nodes[1].setban("192.168.0.1", "add", 1) # ban for 1 seconds + self.nodes[1].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) # ban for 1000 seconds + listBeforeShutdown = self.nodes[1].listbanned() + assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) + # Move time forward by 3 seconds so the third ban has expired + self.nodes[1].setmocktime(old_time + 3) + assert_equal(len(self.nodes[1].listbanned()), 3) + + self.stop_node(1) + + self.nodes[1] = self.start_node(1, self.options.tmpdir) + listAfterShutdown = self.nodes[1].listbanned() + assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) + assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) + assert_equal("/19" in listAfterShutdown[2]['address'], True) + + # Clear ban lists + self.nodes[1].clearbanned() + connect_nodes_bi(self.nodes, 0, 1) + + self.log.info("Test disconnectnode RPCs") + + self.log.info("disconnectnode: fail to disconnect when calling with address and nodeid") + address1 = self.nodes[0].getpeerinfo()[0]['addr'] + node1 = self.nodes[0].getpeerinfo()[0]['addr'] + assert_raises_jsonrpc(-32602, "Only one of address and nodeid should be provided.", self.nodes[0].disconnectnode, address=address1, nodeid=node1) + + self.log.info("disconnectnode: fail to disconnect when calling with junk address") + assert_raises_jsonrpc(-29, "Node not found in connected nodes", self.nodes[0].disconnectnode, address="221B Baker Street") + + self.log.info("disconnectnode: successfully disconnect node by address") + address1 = self.nodes[0].getpeerinfo()[0]['addr'] + self.nodes[0].disconnectnode(address=address1) + assert wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] + + self.log.info("disconnectnode: successfully reconnect node") + connect_nodes_bi(self.nodes, 0, 1) # reconnect the node + assert_equal(len(self.nodes[0].getpeerinfo()), 2) + assert [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] + + self.log.info("disconnectnode: successfully disconnect node by node id") + id1 = self.nodes[0].getpeerinfo()[0]['id'] + self.nodes[0].disconnectnode(nodeid=id1) + assert wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) + assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] + +if __name__ == '__main__': + DisconnectBanTest().main() diff --git a/test/functional/forknotify.py b/test/functional/forknotify.py index 7a365438cc..3bcf0a6795 100755 --- a/test/functional/forknotify.py +++ b/test/functional/forknotify.py @@ -16,21 +16,18 @@ class ForkNotifyTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = False - alert_filename = None # Set by setup_network - def setup_network(self): self.nodes = [] self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") with open(self.alert_filename, 'w', encoding='utf8'): pass # Just open then close to create zero-length file - self.nodes.append(start_node(0, self.options.tmpdir, + self.nodes.append(self.start_node(0, self.options.tmpdir, ["-blockversion=2", "-alertnotify=echo %s >> \"" + self.alert_filename + "\""])) # Node1 mines block.version=211 blocks - self.nodes.append(start_node(1, self.options.tmpdir, + self.nodes.append(self.start_node(1, self.options.tmpdir, ["-blockversion=211"])) connect_nodes(self.nodes[1], 0) - self.is_network_split = False self.sync_all() def run_test(self): diff --git a/test/functional/fundrawtransaction.py b/test/functional/fundrawtransaction.py index b86ea2d877..c41afe2b93 100755 --- a/test/functional/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -23,15 +23,12 @@ class RawTransactionsTest(BitcoinTestFramework): self.num_nodes = 4 def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) + self.setup_nodes() - connect_nodes_bi(self.nodes,0,1) - connect_nodes_bi(self.nodes,1,2) - connect_nodes_bi(self.nodes,0,2) - connect_nodes_bi(self.nodes,0,3) - - self.is_network_split=False - self.sync_all() + connect_nodes_bi(self.nodes, 0, 1) + connect_nodes_bi(self.nodes, 1, 2) + connect_nodes_bi(self.nodes, 0, 2) + connect_nodes_bi(self.nodes, 0, 3) def run_test(self): min_relay_tx_fee = self.nodes[0].getnetworkinfo()['relayfee'] @@ -53,6 +50,11 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].generate(121) self.sync_all() + # ensure that setting changePosition in fundraw with an exact match is handled properly + rawmatch = self.nodes[2].createrawtransaction([], {self.nodes[2].getnewaddress():50}) + rawmatch = self.nodes[2].fundrawtransaction(rawmatch, {"changePosition":1, "subtractFeeFromOutputs":[0]}) + assert_equal(rawmatch["changepos"], -1) + watchonly_address = self.nodes[0].getnewaddress() watchonly_pubkey = self.nodes[0].validateaddress(watchonly_address)["pubkey"] watchonly_amount = Decimal(200) @@ -446,13 +448,13 @@ class RawTransactionsTest(BitcoinTestFramework): ############################################################ # locked wallet test + self.stop_node(0) + self.stop_node(2) + self.stop_node(3) self.nodes[1].encryptwallet("test") self.nodes.pop(1) - stop_node(self.nodes[0], 0) - stop_node(self.nodes[1], 2) - stop_node(self.nodes[2], 3) - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) # This test is not meant to test fee estimation and we'd like # to be sure all txs are sent at a consistent desired feerate for node in self.nodes: @@ -462,7 +464,6 @@ class RawTransactionsTest(BitcoinTestFramework): connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) connect_nodes_bi(self.nodes,0,3) - self.is_network_split=False self.sync_all() # drain the keypool diff --git a/test/functional/getblocktemplate_proposals.py b/test/functional/getblocktemplate_proposals.py index 67745f77d1..fca99c7df5 100755 --- a/test/functional/getblocktemplate_proposals.py +++ b/test/functional/getblocktemplate_proposals.py @@ -73,10 +73,6 @@ class GetBlockTemplateProposalTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = False - def setup_network(self): - self.nodes = self.setup_nodes() - connect_nodes_bi(self.nodes, 0, 1) - def run_test(self): node = self.nodes[0] node.generate(1) # Mine a block to leave initial block download diff --git a/test/functional/getchaintips.py b/test/functional/getchaintips.py index 14222334a6..15f96c565f 100755 --- a/test/functional/getchaintips.py +++ b/test/functional/getchaintips.py @@ -31,7 +31,7 @@ class GetChainTipsTest (BitcoinTestFramework): self.split_network () self.nodes[0].generate(10) self.nodes[2].generate(20) - self.sync_all () + self.sync_all([self.nodes[:2], self.nodes[2:]]) tips = self.nodes[1].getchaintips () assert_equal (len (tips), 1) diff --git a/test/functional/httpbasics.py b/test/functional/httpbasics.py index 8f35f0ab87..4b32e8d9ca 100755 --- a/test/functional/httpbasics.py +++ b/test/functional/httpbasics.py @@ -17,7 +17,7 @@ class HTTPBasicsTest (BitcoinTestFramework): self.setup_clean_chain = False def setup_network(self): - self.nodes = self.setup_nodes() + self.setup_nodes() def run_test(self): diff --git a/test/functional/import-rescan.py b/test/functional/import-rescan.py index 5be095e62d..4fc5078217 100755 --- a/test/functional/import-rescan.py +++ b/test/functional/import-rescan.py @@ -21,7 +21,7 @@ happened previously. from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (start_nodes, connect_nodes, sync_blocks, assert_equal, set_node_times) +from test_framework.util import (connect_nodes, sync_blocks, assert_equal, set_node_times) import collections import enum @@ -121,7 +121,7 @@ class ImportRescanTest(BitcoinTestFramework): if import_node.prune: extra_args[i] += ["-prune=1"] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) for i in range(1, self.num_nodes): connect_nodes(self.nodes[i], 0) diff --git a/test/functional/importmulti.py b/test/functional/importmulti.py index e049302632..e83e85de13 100755 --- a/test/functional/importmulti.py +++ b/test/functional/importmulti.py @@ -12,9 +12,8 @@ class ImportMultiTest (BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = True - def setup_network(self, split=False): - self.nodes = start_nodes(2, self.options.tmpdir) - self.is_network_split=False + def setup_network(self): + self.setup_nodes() def run_test (self): self.log.info("Mining blocks...") @@ -429,8 +428,8 @@ class ImportMultiTest (BitcoinTestFramework): # restart nodes to check for proper serialization/deserialization of watch only address - stop_nodes(self.nodes) - self.nodes = start_nodes(2, self.options.tmpdir) + self.stop_nodes() + self.nodes = self.start_nodes(2, self.options.tmpdir) address_assert = self.nodes[1].validateaddress(watchonly_address) assert_equal(address_assert['iswatchonly'], True) assert_equal(address_assert['ismine'], False) diff --git a/test/functional/importprunedfunds.py b/test/functional/importprunedfunds.py index b4c8ee6c70..94753fe431 100755 --- a/test/functional/importprunedfunds.py +++ b/test/functional/importprunedfunds.py @@ -14,12 +14,6 @@ class ImportPrunedFundsTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 2 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - connect_nodes_bi(self.nodes,0,1) - self.is_network_split=False - self.sync_all() - def run_test(self): self.log.info("Mining blocks...") self.nodes[0].generate(101) diff --git a/test/functional/invalidateblock.py b/test/functional/invalidateblock.py index 8c80b64003..c499d57b90 100755 --- a/test/functional/invalidateblock.py +++ b/test/functional/invalidateblock.py @@ -8,20 +8,15 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * class InvalidateTest(BitcoinTestFramework): - - + def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 def setup_network(self): - self.nodes = [] - self.is_network_split = False - self.nodes.append(start_node(0, self.options.tmpdir)) - self.nodes.append(start_node(1, self.options.tmpdir)) - self.nodes.append(start_node(2, self.options.tmpdir)) - + self.setup_nodes() + def run_test(self): self.log.info("Make sure we repopulate setBlockIndexCandidates after InvalidateBlock:") self.log.info("Mine 4 blocks on Node 0") diff --git a/test/functional/keypool.py b/test/functional/keypool.py index cb9ab688d1..f23a427d1f 100755 --- a/test/functional/keypool.py +++ b/test/functional/keypool.py @@ -20,7 +20,7 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0].encryptwallet('test') bitcoind_processes[0].wait() # Restart node 0 - nodes[0] = start_node(0, self.options.tmpdir) + nodes[0] = self.start_node(0, self.options.tmpdir) # Keep creating keys addr = nodes[0].getnewaddress() addr_data = nodes[0].validateaddress(addr) @@ -84,8 +84,5 @@ class KeyPoolTest(BitcoinTestFramework): self.setup_clean_chain = False self.num_nodes = 1 - def setup_network(self): - self.nodes = self.setup_nodes() - if __name__ == '__main__': KeyPoolTest().main() diff --git a/test/functional/listsinceblock.py b/test/functional/listsinceblock.py index a75e66c8c4..f3d41e573e 100755 --- a/test/functional/listsinceblock.py +++ b/test/functional/listsinceblock.py @@ -43,7 +43,6 @@ class ListSinceBlockTest (BitcoinTestFramework): This test only checks that [tx0] is present. ''' - assert_equal(self.is_network_split, False) self.nodes[2].generate(101) self.sync_all() @@ -54,7 +53,6 @@ class ListSinceBlockTest (BitcoinTestFramework): # Split network into two self.split_network() - assert_equal(self.is_network_split, True) # send to nodes[0] from nodes[2] senttx = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) @@ -64,7 +62,7 @@ class ListSinceBlockTest (BitcoinTestFramework): self.nodes[2].generate(7) self.log.info('lastblockhash=%s' % (lastblockhash)) - self.sync_all() + self.sync_all([self.nodes[:2], self.nodes[2:]]) self.join_network() diff --git a/test/functional/listtransactions.py b/test/functional/listtransactions.py index 68d14093ce..f69f1c5724 100755 --- a/test/functional/listtransactions.py +++ b/test/functional/listtransactions.py @@ -24,7 +24,7 @@ class ListTransactionsTest(BitcoinTestFramework): def setup_nodes(self): #This test requires mocktime enable_mocktime() - return start_nodes(self.num_nodes, self.options.tmpdir) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) def run_test(self): # Simple send, 0 to 1: diff --git a/test/functional/maxblocksinflight.py b/test/functional/maxblocksinflight.py deleted file mode 100755 index 4ef2a35a44..0000000000 --- a/test/functional/maxblocksinflight.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test nodes responses to having many blocks in flight. - -In this test we connect to one node over p2p, send it numerous inv's, and -compare the resulting number of getdata requests to a max allowed value. We -test for exceeding 128 blocks in flight, which was the limit an 0.9 client will -reach. [0.10 clients shouldn't request more than 16 from a single peer.] -""" - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -MAX_REQUESTS = 128 - -class TestManager(NodeConnCB): - def on_getdata(self, conn, message): - self.log.debug("got getdata %s" % repr(message)) - # Log the requests - for inv in message.inv: - if inv.hash not in self.blockReqCounts: - self.blockReqCounts[inv.hash] = 0 - self.blockReqCounts[inv.hash] += 1 - - def on_close(self, conn): - if not self.disconnectOkay: - raise EarlyDisconnectError(0) - - def add_new_connection(self, connection): - super().add_connection(connection) - self.blockReqCounts = {} - self.disconnectOkay = False - - def run(self): - self.connection.rpc.generate(1) # Leave IBD - - numBlocksToGenerate = [8, 16, 128, 1024] - for count in range(len(numBlocksToGenerate)): - current_invs = [] - for i in range(numBlocksToGenerate[count]): - current_invs.append(CInv(2, random.randrange(0, 1 << 256))) - if len(current_invs) >= 50000: - self.connection.send_message(msg_inv(current_invs)) - current_invs = [] - if len(current_invs) > 0: - self.connection.send_message(msg_inv(current_invs)) - - # Wait and see how many blocks were requested - time.sleep(2) - - total_requests = 0 - with mininode_lock: - for key in self.blockReqCounts: - total_requests += self.blockReqCounts[key] - if self.blockReqCounts[key] > 1: - raise AssertionError("Error, test failed: block %064x requested more than once" % key) - if total_requests > MAX_REQUESTS: - raise AssertionError("Error, too many blocks (%d) requested" % total_requests) - self.log.info("Round %d: success (total requests: %d)" % (count, total_requests)) - - self.disconnectOkay = True - self.connection.disconnect_node() - - -class MaxBlocksInFlightTest(BitcoinTestFramework): - def add_options(self, parser): - parser.add_option("--testbinary", dest="testbinary", - default=os.getenv("BITCOIND", "bitcoind"), - help="Binary to test max block requests behavior") - - def __init__(self): - super().__init__() - self.setup_clean_chain = True - self.num_nodes = 1 - - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-whitelist=127.0.0.1']], - binary=[self.options.testbinary]) - - def run_test(self): - test = TestManager() - # pass log handler through to the test manager object - test.log = self.log - test.add_new_connection(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test)) - NetworkThread().start() # Start up network handling in another thread - test.run() - -if __name__ == '__main__': - MaxBlocksInFlightTest().main() diff --git a/test/functional/maxuploadtarget.py b/test/functional/maxuploadtarget.py index 9b42bf276c..66e5bd29e6 100755 --- a/test/functional/maxuploadtarget.py +++ b/test/functional/maxuploadtarget.py @@ -10,63 +10,24 @@ if uploadtarget has been reached. if uploadtarget has been reached. * Verify that the upload counters are reset after 24 hours. """ +from collections import defaultdict +import time from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -import time -# TestNode: bare-bones "peer". Used mostly as a conduit for a test to sending -# p2p messages to a node, generating the messages in the main testing logic. class TestNode(NodeConnCB): def __init__(self): super().__init__() - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong() - self.block_receive_map = {} - - def add_connection(self, conn): - self.connection = conn - self.peer_disconnected = False + self.block_receive_map = defaultdict(int) def on_inv(self, conn, message): pass - # Track the last getdata message we receive (used in the test) - def on_getdata(self, conn, message): - self.last_getdata = message - def on_block(self, conn, message): message.block.calc_sha256() - try: - self.block_receive_map[message.block.sha256] += 1 - except KeyError as e: - self.block_receive_map[message.block.sha256] = 1 - - # Spin until verack message is received from the node. - # We use this to signal that our test can begin. This - # is called from the testing thread, so it needs to acquire - # the global lock. - def wait_for_verack(self): - def veracked(): - return self.verack_received - return wait_until(veracked, timeout=10) - - def wait_for_disconnect(self): - def disconnected(): - return self.peer_disconnected - return wait_until(disconnected, timeout=10) - - # Wrapper for the NodeConn's send_message function - def send_message(self, message): - self.connection.send_message(message) - - def on_pong(self, conn, message): - self.last_pong = message - - def on_close(self, conn): - self.peer_disconnected = True + self.block_receive_map[message.block.sha256] += 1 class MaxUploadTest(BitcoinTestFramework): @@ -74,15 +35,11 @@ class MaxUploadTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 1 + self.extra_args = [["-maxuploadtarget=800", "-blockmaxsize=999000"]] # Cache for utxos, as the listunspent may take a long time later in the test self.utxo_cache = [] - def setup_network(self): - # Start a node with maxuploadtarget of 200 MB (/24h) - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxuploadtarget=800", "-blockmaxsize=999000"])) - def run_test(self): # Before we connect anything, we first set the time on the node # to be in the past, otherwise things break because the CNode @@ -189,36 +146,29 @@ class MaxUploadTest(BitcoinTestFramework): #stop and start node 0 with 1MB maxuploadtarget, whitelist 127.0.0.1 self.log.info("Restarting nodes with -whitelist=127.0.0.1") - stop_node(self.nodes[0], 0) - self.nodes[0] = start_node(0, self.options.tmpdir, ["-whitelist=127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000"]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-whitelist=127.0.0.1", "-maxuploadtarget=1", "-blockmaxsize=999000"]) - #recreate/reconnect 3 test nodes - test_nodes = [] - connections = [] - - for i in range(3): - test_nodes.append(TestNode()) - connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[i])) - test_nodes[i].add_connection(connections[i]) + #recreate/reconnect a test node + test_nodes = [TestNode()] + connections = [NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_nodes[0])] + test_nodes[0].add_connection(connections[0]) NetworkThread().start() # Start up network handling in another thread - [x.wait_for_verack() for x in test_nodes] + test_nodes[0].wait_for_verack() #retrieve 20 blocks which should be enough to break the 1MB limit getdata_request.inv = [CInv(2, big_new_block)] for i in range(20): - test_nodes[1].send_message(getdata_request) - test_nodes[1].sync_with_ping() - assert_equal(test_nodes[1].block_receive_map[big_new_block], i+1) + test_nodes[0].send_message(getdata_request) + test_nodes[0].sync_with_ping() + assert_equal(test_nodes[0].block_receive_map[big_new_block], i+1) getdata_request.inv = [CInv(2, big_old_block)] - test_nodes[1].send_message(getdata_request) - test_nodes[1].wait_for_disconnect() - assert_equal(len(self.nodes[0].getpeerinfo()), 3) #node is still connected because of the whitelist + test_nodes[0].send_and_ping(getdata_request) + assert_equal(len(self.nodes[0].getpeerinfo()), 1) #node is still connected because of the whitelist - self.log.info("Peer 1 still connected after trying to download old block (whitelisted)") - - [c.disconnect_node() for c in connections] + self.log.info("Peer still connected after trying to download old block (whitelisted)") if __name__ == '__main__': MaxUploadTest().main() diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index a7ca576aee..2777291dd0 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -9,30 +9,25 @@ from test_framework.util import * class MempoolLimitTest(BitcoinTestFramework): - def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxmempool=5", "-spendzeroconfchange=0"])) - self.is_network_split = False - self.sync_all() - self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] - def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 1 - - self.txouts = gen_return_txouts() + self.extra_args = [["-maxmempool=5", "-spendzeroconfchange=0"]] def run_test(self): + txouts = gen_return_txouts() + relayfee = self.nodes[0].getnetworkinfo()['relayfee'] + txids = [] - utxos = create_confirmed_utxos(self.relayfee, self.nodes[0], 91) + utxos = create_confirmed_utxos(relayfee, self.nodes[0], 91) #create a mempool tx that will be evicted us0 = utxos.pop() inputs = [{ "txid" : us0["txid"], "vout" : us0["vout"]}] outputs = {self.nodes[0].getnewaddress() : 0.0001} tx = self.nodes[0].createrawtransaction(inputs, outputs) - self.nodes[0].settxfee(self.relayfee) # specifically fund this tx with low fee + self.nodes[0].settxfee(relayfee) # specifically fund this tx with low fee txF = self.nodes[0].fundrawtransaction(tx) self.nodes[0].settxfee(0) # return to automatic fee selection txFS = self.nodes[0].signrawtransaction(txF['hex']) @@ -42,7 +37,7 @@ class MempoolLimitTest(BitcoinTestFramework): base_fee = relayfee*100 for i in range (3): txids.append([]) - txids[i] = create_lots_of_big_transactions(self.nodes[0], self.txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) + txids[i] = create_lots_of_big_transactions(self.nodes[0], txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) # by now, the tx should be evicted, check confirmation state assert(txid not in self.nodes[0].getrawmempool()) diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index feec8a7fd9..72f04095f4 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -16,14 +16,7 @@ class MempoolPackagesTest(BitcoinTestFramework): super().__init__() self.num_nodes = 2 self.setup_clean_chain = False - - def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000"])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-maxorphantx=1000", "-limitancestorcount=5"])) - connect_nodes(self.nodes[0], 1) - self.is_network_split = False - self.sync_all() + self.extra_args = [["-maxorphantx=1000"], ["-maxorphantx=1000", "-limitancestorcount=5"]] # Build a transaction that spends parent_txid:vout # Return amount sent diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py new file mode 100755 index 0000000000..e0889fd5e9 --- /dev/null +++ b/test/functional/mempool_persist.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test mempool persistence. + +By default, bitcoind will dump mempool on shutdown and +then reload it on startup. This can be overridden with +the -persistmempool=0 command line option. + +Test is as follows: + + - start node0, node1 and node2. node1 has -persistmempool=0 + - create 5 transactions on node2 to its own address. Note that these + are not sent to node0 or node1 addresses because we don't want + them to be saved in the wallet. + - check that node0 and node1 have 5 transactions in their mempools + - shutdown all nodes. + - startup node0. Verify that it still has 5 transactions + in its mempool. Shutdown node0. This tests that by default the + mempool is persistent. + - startup node1. Verify that its mempool is empty. Shutdown node1. + This tests that with -persistmempool=0, the mempool is not + dumped to disk when the node is shut down. + - Restart node0 with -persistmempool=0. Verify that its mempool is + empty. Shutdown node0. This tests that with -persistmempool=0, + the mempool is not loaded from disk on start up. + - Restart node0 with -persistmempool. Verify that it has 5 + transactions in its mempool. This tests that -persistmempool=0 + does not overwrite a previously valid mempool stored on disk. + +""" +import time + +from test_framework.mininode import wait_until +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +class MempoolPersistTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + # We need 3 nodes for this test. Node1 does not have a persistent mempool. + self.num_nodes = 3 + self.setup_clean_chain = False + self.extra_args = [[], ["-persistmempool=0"], []] + + def run_test(self): + chain_height = self.nodes[0].getblockcount() + assert_equal(chain_height, 200) + + self.log.debug("Mine a single block to get out of IBD") + self.nodes[0].generate(1) + self.sync_all() + + self.log.debug("Send 5 transactions from node2 (to its own address)") + for i in range(5): + self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) + self.sync_all() + + self.log.debug("Verify that node0 and node1 have 5 transactions in their mempools") + assert_equal(len(self.nodes[0].getrawmempool()), 5) + assert_equal(len(self.nodes[1].getrawmempool()), 5) + + self.log.debug("Stop-start node0 and node1. Verify that node0 has the transactions in its mempool and node1 does not.") + self.stop_nodes() + self.nodes = [] + self.nodes.append(self.start_node(0, self.options.tmpdir)) + self.nodes.append(self.start_node(1, self.options.tmpdir)) + # Give bitcoind a second to reload the mempool + time.sleep(1) + assert wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5) + assert_equal(len(self.nodes[1].getrawmempool()), 0) + + self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") + self.stop_nodes() + self.nodes = [] + self.nodes.append(self.start_node(0, self.options.tmpdir, ["-persistmempool=0"])) + # Give bitcoind a second to reload the mempool + time.sleep(1) + assert_equal(len(self.nodes[0].getrawmempool()), 0) + + self.log.debug("Stop-start node0. Verify that it has the transactions in its mempool.") + self.stop_nodes() + self.nodes = [] + self.nodes.append(self.start_node(0, self.options.tmpdir)) + assert wait_until(lambda: len(self.nodes[0].getrawmempool()) == 5) + +if __name__ == '__main__': + MempoolPersistTest().main() diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 812b54ffcb..937bf4bab5 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -17,18 +17,10 @@ class MempoolCoinbaseTest(BitcoinTestFramework): super().__init__() self.num_nodes = 2 self.setup_clean_chain = False + self.extra_args = [["-checkmempool"]] * 2 alert_filename = None # Set by setup_network - def setup_network(self): - args = ["-checkmempool"] - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, args)) - self.nodes.append(start_node(1, self.options.tmpdir, args)) - connect_nodes(self.nodes[1], 0) - self.is_network_split = False - self.sync_all() - def run_test(self): # Start with a 200 block chain assert_equal(self.nodes[0].getblockcount(), 200) diff --git a/test/functional/mempool_resurrect_test.py b/test/functional/mempool_resurrect_test.py index 727892d1f2..a2f6228df9 100755 --- a/test/functional/mempool_resurrect_test.py +++ b/test/functional/mempool_resurrect_test.py @@ -14,13 +14,8 @@ class MempoolCoinbaseTest(BitcoinTestFramework): super().__init__() self.num_nodes = 1 self.setup_clean_chain = False - - def setup_network(self): # Just need one node for this test - args = ["-checkmempool"] - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, args)) - self.is_network_split = False + self.extra_args = [["-checkmempool"]] def run_test(self): node0_address = self.nodes[0].getnewaddress() diff --git a/test/functional/mempool_spendcoinbase.py b/test/functional/mempool_spendcoinbase.py index f562a93d86..277ea45ad5 100755 --- a/test/functional/mempool_spendcoinbase.py +++ b/test/functional/mempool_spendcoinbase.py @@ -22,13 +22,7 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework): super().__init__() self.num_nodes = 1 self.setup_clean_chain = False - - def setup_network(self): - # Just need one node for this test - args = ["-checkmempool"] - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, args)) - self.is_network_split = False + self.extra_args = [["-checkmempool"]] def run_test(self): chain_height = self.nodes[0].getblockcount() diff --git a/test/functional/merkle_blocks.py b/test/functional/merkle_blocks.py index 5963f2e7b6..06af72ef10 100755 --- a/test/functional/merkle_blocks.py +++ b/test/functional/merkle_blocks.py @@ -13,20 +13,15 @@ class MerkleBlockTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 4 + # Nodes 0/1 are "wallet" nodes, Nodes 2/3 are used for testing + self.extra_args = [[], [], [], ["-txindex"]] def setup_network(self): - self.nodes = [] - # Nodes 0/1 are "wallet" nodes - self.nodes.append(start_node(0, self.options.tmpdir)) - self.nodes.append(start_node(1, self.options.tmpdir)) - # Nodes 2/3 are used for testing - self.nodes.append(start_node(2, self.options.tmpdir)) - self.nodes.append(start_node(3, self.options.tmpdir, ["-txindex"])) + self.setup_nodes() connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[0], 3) - self.is_network_split = False self.sync_all() def run_test(self): diff --git a/test/functional/multi_rpc.py b/test/functional/multi_rpc.py index a9701c548b..6ff91a960b 100755 --- a/test/functional/multi_rpc.py +++ b/test/functional/multi_rpc.py @@ -27,9 +27,6 @@ class HTTPBasicsTest (BitcoinTestFramework): f.write(rpcauth+"\n") f.write(rpcauth2+"\n") - def setup_network(self): - self.nodes = self.setup_nodes() - def run_test(self): ################################################## diff --git a/test/functional/net.py b/test/functional/net.py index e9463c7dc7..3ba3764cf9 100755 --- a/test/functional/net.py +++ b/test/functional/net.py @@ -7,15 +7,14 @@ Tests correspond to code in rpc/net.cpp. """ -from decimal import Decimal import time from test_framework.test_framework import BitcoinTestFramework -from test_framework.authproxy import JSONRPCException from test_framework.util import ( assert_equal, - start_nodes, + assert_raises_jsonrpc, connect_nodes_bi, + p2p_port, ) @@ -25,15 +24,43 @@ class NetTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 2 - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - connect_nodes_bi(self.nodes, 0, 1) - self.is_network_split = False - self.sync_all() - def run_test(self): + self._test_connection_count() + self._test_getnettotals() + self._test_getnetworkinginfo() + self._test_getaddednodeinfo() + self._test_getpeerinfo() + + def _test_connection_count(self): + # connect_nodes_bi connects each node to the other + assert_equal(self.nodes[0].getconnectioncount(), 2) + + def _test_getnettotals(self): + # check that getnettotals totalbytesrecv and totalbytessent + # are consistent with getpeerinfo + peer_info = self.nodes[0].getpeerinfo() + assert_equal(len(peer_info), 2) + net_totals = self.nodes[0].getnettotals() + assert_equal(sum([peer['bytesrecv'] for peer in peer_info]), + net_totals['totalbytesrecv']) + assert_equal(sum([peer['bytessent'] for peer in peer_info]), + net_totals['totalbytessent']) + # test getnettotals and getpeerinfo by doing a ping + # the bytes sent/received should change + # note ping and pong are 32 bytes each + self.nodes[0].ping() + time.sleep(0.1) + peer_info_after_ping = self.nodes[0].getpeerinfo() + net_totals_after_ping = self.nodes[0].getnettotals() + for before, after in zip(peer_info, peer_info_after_ping): + assert_equal(before['bytesrecv_per_msg']['pong'] + 32, after['bytesrecv_per_msg']['pong']) + assert_equal(before['bytessent_per_msg']['ping'] + 32, after['bytessent_per_msg']['ping']) + assert_equal(net_totals['totalbytesrecv'] + 32*2, net_totals_after_ping['totalbytesrecv']) + assert_equal(net_totals['totalbytessent'] + 32*2, net_totals_after_ping['totalbytessent']) + + def _test_getnetworkinginfo(self): assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) - assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) # bilateral connection + assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) self.nodes[0].setnetworkactive(False) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False) @@ -49,6 +76,25 @@ class NetTest(BitcoinTestFramework): assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) + def _test_getaddednodeinfo(self): + assert_equal(self.nodes[0].getaddednodeinfo(), []) + # add a node (node2) to node0 + ip_port = "127.0.0.1:{}".format(p2p_port(2)) + self.nodes[0].addnode(ip_port, 'add') + # check that the node has indeed been added + added_nodes = self.nodes[0].getaddednodeinfo(ip_port) + assert_equal(len(added_nodes), 1) + assert_equal(added_nodes[0]['addednode'], ip_port) + # check that a non-existant node returns an error + assert_raises_jsonrpc(-24, "Node has not been added", + self.nodes[0].getaddednodeinfo, '1.1.1.1') + + def _test_getpeerinfo(self): + peer_info = [x.getpeerinfo() for x in self.nodes] + # check both sides of bidirectional connection between nodes + # the address bound to on one side will be the source address for the other node + assert_equal(peer_info[0][0]['addrbind'], peer_info[1][0]['addr']) + assert_equal(peer_info[1][0]['addrbind'], peer_info[0][0]['addr']) if __name__ == '__main__': NetTest().main() diff --git a/test/functional/nodehandling.py b/test/functional/nodehandling.py deleted file mode 100755 index a6b10a0d83..0000000000 --- a/test/functional/nodehandling.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test node handling.""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -import urllib.parse - -class NodeHandlingTest (BitcoinTestFramework): - - def __init__(self): - super().__init__() - self.num_nodes = 4 - self.setup_clean_chain = False - - def run_test(self): - ########################### - # setban/listbanned tests # - ########################### - assert_equal(len(self.nodes[2].getpeerinfo()), 4) #we should have 4 nodes at this point - self.nodes[2].setban("127.0.0.1", "add") - time.sleep(3) #wait till the nodes are disconected - assert_equal(len(self.nodes[2].getpeerinfo()), 0) #all nodes must be disconnected at this point - assert_equal(len(self.nodes[2].listbanned()), 1) - self.nodes[2].clearbanned() - assert_equal(len(self.nodes[2].listbanned()), 0) - self.nodes[2].setban("127.0.0.0/24", "add") - assert_equal(len(self.nodes[2].listbanned()), 1) - # This will throw an exception because 127.0.0.1 is within range 127.0.0.0/24 - assert_raises_jsonrpc(-23, "IP/Subnet already banned", self.nodes[2].setban, "127.0.0.1", "add") - # This will throw an exception because 127.0.0.1/42 is not a real subnet - assert_raises_jsonrpc(-30, "Error: Invalid IP/Subnet", self.nodes[2].setban, "127.0.0.1/42", "add") - assert_equal(len(self.nodes[2].listbanned()), 1) #still only one banned ip because 127.0.0.1 is within the range of 127.0.0.0/24 - # This will throw an exception because 127.0.0.1 was not added above - assert_raises_jsonrpc(-30, "Error: Unban failed", self.nodes[2].setban, "127.0.0.1", "remove") - assert_equal(len(self.nodes[2].listbanned()), 1) - self.nodes[2].setban("127.0.0.0/24", "remove") - assert_equal(len(self.nodes[2].listbanned()), 0) - self.nodes[2].clearbanned() - assert_equal(len(self.nodes[2].listbanned()), 0) - - ##test persisted banlist - self.nodes[2].setban("127.0.0.0/32", "add") - self.nodes[2].setban("127.0.0.0/24", "add") - self.nodes[2].setban("192.168.0.1", "add", 1) #ban for 1 seconds - self.nodes[2].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) #ban for 1000 seconds - listBeforeShutdown = self.nodes[2].listbanned() - assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) #must be here - time.sleep(2) #make 100% sure we expired 192.168.0.1 node time - - #stop node - stop_node(self.nodes[2], 2) - - self.nodes[2] = start_node(2, self.options.tmpdir) - listAfterShutdown = self.nodes[2].listbanned() - assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) - assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) - assert_equal("/19" in listAfterShutdown[2]['address'], True) - - ########################### - # RPC disconnectnode test # - ########################### - url = urllib.parse.urlparse(self.nodes[1].url) - self.nodes[0].disconnectnode(url.hostname+":"+str(p2p_port(1))) - time.sleep(2) #disconnecting a node needs a little bit of time - for node in self.nodes[0].getpeerinfo(): - assert(node['addr'] != url.hostname+":"+str(p2p_port(1))) - - connect_nodes_bi(self.nodes,0,1) #reconnect the node - found = False - for node in self.nodes[0].getpeerinfo(): - if node['addr'] == url.hostname+":"+str(p2p_port(1)): - found = True - assert(found) - -if __name__ == '__main__': - NodeHandlingTest ().main () diff --git a/test/functional/nulldummy.py b/test/functional/nulldummy.py index 369c593a90..9717add272 100755 --- a/test/functional/nulldummy.py +++ b/test/functional/nulldummy.py @@ -41,11 +41,7 @@ class NULLDUMMYTest(BitcoinTestFramework): super().__init__() self.num_nodes = 1 self.setup_clean_chain = True - - def setup_network(self): - # Must set the blockversion for this test - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-whitelist=127.0.0.1', '-walletprematurewitness']]) + self.extra_args = [['-whitelist=127.0.0.1', '-walletprematurewitness']] def run_test(self): self.address = self.nodes[0].getnewaddress() diff --git a/test/functional/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py index c09945baa6..322cb767db 100755 --- a/test/functional/p2p-acceptblock.py +++ b/test/functional/p2p-acceptblock.py @@ -54,40 +54,6 @@ from test_framework.util import * import time from test_framework.blocktools import create_block, create_coinbase -# TestNode: bare-bones "peer". Used mostly as a conduit for a test to sending -# p2p messages to a node, generating the messages in the main testing logic. -class TestNode(NodeConnCB): - def __init__(self): - super().__init__() - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong() - - def add_connection(self, conn): - self.connection = conn - - # Track the last getdata message we receive (used in the test) - def on_getdata(self, conn, message): - self.last_getdata = message - - # Spin until verack message is received from the node. - # We use this to signal that our test can begin. This - # is called from the testing thread, so it needs to acquire - # the global lock. - def wait_for_verack(self): - while True: - with mininode_lock: - if self.verack_received: - return - time.sleep(0.05) - - # Wrapper for the NodeConn's send_message function - def send_message(self, message): - self.connection.send_message(message) - - def on_pong(self, conn, message): - self.last_pong = message - class AcceptBlockTest(BitcoinTestFramework): def add_options(self, parser): parser.add_option("--testbinary", dest="testbinary", @@ -98,22 +64,18 @@ class AcceptBlockTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 2 + self.extra_args = [[], ["-whitelist=127.0.0.1"]] def setup_network(self): # Node0 will be used to test behavior of processing unrequested blocks # from peers which are not whitelisted, while Node1 will be used for # the whitelisted case. - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, - binary=self.options.testbinary)) - self.nodes.append(start_node(1, self.options.tmpdir, - ["-whitelist=127.0.0.1"], - binary=self.options.testbinary)) + self.setup_nodes() def run_test(self): # Setup the p2p connections and start up the network thread. - test_node = TestNode() # connects to node0 (not whitelisted) - white_node = TestNode() # connects to node1 (whitelisted) + test_node = NodeConnCB() # connects to node0 (not whitelisted) + white_node = NodeConnCB() # connects to node1 (whitelisted) connections = [] connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test_node)) @@ -238,12 +200,12 @@ class AcceptBlockTest(BitcoinTestFramework): # triggers a getdata on block 2 (it should if block 2 is missing). with mininode_lock: # Clear state so we can check the getdata request - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) test_node.send_message(msg_inv([CInv(2, blocks_h3[0].sha256)])) test_node.sync_with_ping() with mininode_lock: - getdata = test_node.last_getdata + getdata = test_node.last_message["getdata"] # Check that the getdata includes the right block assert_equal(getdata.inv[0].hash, blocks_h2f[0].sha256) diff --git a/test/functional/p2p-compactblocks.py b/test/functional/p2p-compactblocks.py index bf8d113767..9b302120ac 100755 --- a/test/functional/p2p-compactblocks.py +++ b/test/functional/p2p-compactblocks.py @@ -19,64 +19,31 @@ class TestNode(NodeConnCB): def __init__(self): super().__init__() self.last_sendcmpct = [] - self.last_headers = None - self.last_inv = None - self.last_cmpctblock = None self.block_announced = False - self.last_getdata = None - self.last_getheaders = None - self.last_getblocktxn = None - self.last_block = None - self.last_blocktxn = None # Store the hashes of blocks we've seen announced. # This is for synchronizing the p2p message traffic, # so we can eg wait until a particular block is announced. - self.set_announced_blockhashes = set() - self.connected = False - - def on_open(self, conn): - self.connected = True - - def on_close(self, conn): - self.connected = False + self.announced_blockhashes = set() def on_sendcmpct(self, conn, message): self.last_sendcmpct.append(message) - def on_block(self, conn, message): - self.last_block = message - def on_cmpctblock(self, conn, message): - self.last_cmpctblock = message self.block_announced = True - self.last_cmpctblock.header_and_shortids.header.calc_sha256() - self.set_announced_blockhashes.add(self.last_cmpctblock.header_and_shortids.header.sha256) + self.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() + self.announced_blockhashes.add(self.last_message["cmpctblock"].header_and_shortids.header.sha256) def on_headers(self, conn, message): - self.last_headers = message self.block_announced = True - for x in self.last_headers.headers: + for x in self.last_message["headers"].headers: x.calc_sha256() - self.set_announced_blockhashes.add(x.sha256) + self.announced_blockhashes.add(x.sha256) def on_inv(self, conn, message): - self.last_inv = message - for x in self.last_inv.inv: + for x in self.last_message["inv"].inv: if x.type == 2: self.block_announced = True - self.set_announced_blockhashes.add(x.hash) - - def on_getdata(self, conn, message): - self.last_getdata = message - - def on_getheaders(self, conn, message): - self.last_getheaders = message - - def on_getblocktxn(self, conn, message): - self.last_getblocktxn = message - - def on_blocktxn(self, conn, message): - self.last_blocktxn = message + self.announced_blockhashes.add(x.hash) # Requires caller to hold mininode_lock def received_block_announcement(self): @@ -85,9 +52,9 @@ class TestNode(NodeConnCB): def clear_block_announcement(self): with mininode_lock: self.block_announced = False - self.last_inv = None - self.last_headers = None - self.last_cmpctblock = None + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) + self.last_message.pop("cmpctblock", None) def get_headers(self, locator, hashstop): msg = msg_getheaders() @@ -103,15 +70,14 @@ class TestNode(NodeConnCB): def request_headers_and_sync(self, locator, hashstop=0): self.clear_block_announcement() self.get_headers(locator, hashstop) - assert(wait_until(self.received_block_announcement, timeout=30)) - assert(self.received_block_announcement()) + assert wait_until(self.received_block_announcement, timeout=30) self.clear_block_announcement() # Block until a block announcement for a particular block hash is # received. def wait_for_block_announcement(self, block_hash, timeout=30): def received_hash(): - return (block_hash in self.set_announced_blockhashes) + return (block_hash in self.announced_blockhashes) return wait_until(received_hash, timeout=timeout) def send_await_disconnect(self, message, timeout=30): @@ -132,17 +98,9 @@ class CompactBlocksTest(BitcoinTestFramework): self.setup_clean_chain = True # Node0 = pre-segwit, node1 = segwit-aware self.num_nodes = 2 + self.extra_args = [["-bip9params=segwit:0:0"], ["-txindex"]] self.utxos = [] - def setup_network(self): - self.nodes = [] - - # Start up node0 to be a version 1, pre-segwit node. - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - [["-bip9params=segwit:0:0"], - ["-txindex"]]) - connect_nodes(self.nodes[0], 1) - def build_block_on_tip(self, node, segwit=False): height = node.getblockcount() tip = node.getbestblockhash() @@ -214,14 +172,14 @@ class CompactBlocksTest(BitcoinTestFramework): with mininode_lock: assert predicate(peer), ( "block_hash={!r}, cmpctblock={!r}, inv={!r}".format( - block_hash, peer.last_cmpctblock, peer.last_inv)) + block_hash, peer.last_message.get("cmpctblock", None), peer.last_message.get("inv", None))) # We shouldn't get any block announcements via cmpctblock yet. - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) # Try one more time, this time after requesting headers. test_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_inv is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "inv" in p.last_message) # Test a few ways of using sendcmpct that should NOT # result in compact block announcements. @@ -233,7 +191,7 @@ class CompactBlocksTest(BitcoinTestFramework): sendcmpct.version = preferred_version+1 sendcmpct.announce = True test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) # Headers sync before next test. test_node.request_headers_and_sync(locator=[tip]) @@ -242,7 +200,7 @@ class CompactBlocksTest(BitcoinTestFramework): sendcmpct.version = preferred_version sendcmpct.announce = False test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message) # Headers sync before next test. test_node.request_headers_and_sync(locator=[tip]) @@ -251,26 +209,26 @@ class CompactBlocksTest(BitcoinTestFramework): sendcmpct.version = preferred_version sendcmpct.announce = True test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Try one more time (no headers sync should be needed!) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Try one more time, after turning on sendheaders test_node.send_and_ping(msg_sendheaders()) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Try one more time, after sending a version-1, announce=false message. sendcmpct.version = preferred_version-1 sendcmpct.announce = False test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" in p.last_message) # Now turn off announcements sendcmpct.version = preferred_version sendcmpct.announce = False test_node.send_and_ping(sendcmpct) - check_announcement_of_new_block(node, test_node, lambda p: p.last_cmpctblock is None and p.last_headers is not None) + check_announcement_of_new_block(node, test_node, lambda p: "cmpctblock" not in p.last_message and "headers" in p.last_message) if old_node is not None: # Verify that a peer using an older protocol version can receive @@ -280,7 +238,7 @@ class CompactBlocksTest(BitcoinTestFramework): old_node.send_and_ping(sendcmpct) # Header sync old_node.request_headers_and_sync(locator=[tip]) - check_announcement_of_new_block(node, old_node, lambda p: p.last_cmpctblock is not None) + check_announcement_of_new_block(node, old_node, lambda p: "cmpctblock" in p.last_message) # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. def test_invalid_cmpctblock_message(self): @@ -345,9 +303,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert(test_node.last_cmpctblock is not None) + assert("cmpctblock" in test_node.last_message) # Convert the on-the-wire representation to absolute indexes - header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids) + header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) # Now fetch the compact block using a normal non-announce getdata @@ -362,9 +320,9 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert(test_node.last_cmpctblock is not None) + assert("cmpctblock" in test_node.last_message) # Convert the on-the-wire representation to absolute indexes - header_and_shortids = HeaderAndShortIDs(test_node.last_cmpctblock.header_and_shortids) + header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) def check_compactblock_construction_from_block(self, version, header_and_shortids, block_hash, block): @@ -424,20 +382,20 @@ class CompactBlocksTest(BitcoinTestFramework): for announce in ["inv", "header"]: block = self.build_block_on_tip(node, segwit=segwit) with mininode_lock: - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) if announce == "inv": test_node.send_message(msg_inv([CInv(2, block.sha256)])) - success = wait_until(lambda: test_node.last_getheaders is not None, timeout=30) + success = wait_until(lambda: "getheaders" in test_node.last_message, timeout=30) assert(success) test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) - success = wait_until(lambda: test_node.last_getdata is not None, timeout=30) + success = wait_until(lambda: "getdata" in test_node.last_message, timeout=30) assert(success) - assert_equal(len(test_node.last_getdata.inv), 1) - assert_equal(test_node.last_getdata.inv[0].type, 4) - assert_equal(test_node.last_getdata.inv[0].hash, block.sha256) + assert_equal(len(test_node.last_message["getdata"].inv), 1) + assert_equal(test_node.last_message["getdata"].inv[0].type, 4) + assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Send back a compactblock message that omits the coinbase comp_block = HeaderAndShortIDs() @@ -453,8 +411,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. with mininode_lock: - assert(test_node.last_getblocktxn is not None) - absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute() + assert("getblocktxn" in test_node.last_message) + absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request # Send the coinbase, and verify that the tip advances. @@ -493,8 +451,8 @@ class CompactBlocksTest(BitcoinTestFramework): msg = msg_cmpctblock(compact_block.to_p2p()) peer.send_and_ping(msg) with mininode_lock: - assert(peer.last_getblocktxn is not None) - absolute_indexes = peer.last_getblocktxn.block_txn_request.to_absolute() + assert("getblocktxn" in peer.last_message) + absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, expected_result) def test_tip_after_message(node, peer, msg, tip): @@ -558,14 +516,14 @@ class CompactBlocksTest(BitcoinTestFramework): # Clear out last request. with mininode_lock: - test_node.last_getblocktxn = None + test_node.last_message.pop("getblocktxn", None) # Send compact block comp_block.initialize_from_block(block, prefill_list=[0], use_witness=with_witness) test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) with mininode_lock: # Shouldn't have gotten a request for any transaction - assert(test_node.last_getblocktxn is None) + assert("getblocktxn" not in test_node.last_message) # Incorrectly responding to a getblocktxn shouldn't cause the block to be # permanently failed. @@ -591,8 +549,8 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] with mininode_lock: - assert(test_node.last_getblocktxn is not None) - absolute_indexes = test_node.last_getblocktxn.block_txn_request.to_absolute() + assert("getblocktxn" in test_node.last_message) + absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) # Now give an incorrect response. @@ -613,11 +571,11 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # We should receive a getdata request - success = wait_until(lambda: test_node.last_getdata is not None, timeout=10) + success = wait_until(lambda: "getdata" in test_node.last_message, timeout=10) assert(success) - assert_equal(len(test_node.last_getdata.inv), 1) - assert(test_node.last_getdata.inv[0].type == 2 or test_node.last_getdata.inv[0].type == 2|MSG_WITNESS_FLAG) - assert_equal(test_node.last_getdata.inv[0].hash, block.sha256) + assert_equal(len(test_node.last_message["getdata"].inv), 1) + assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2|MSG_WITNESS_FLAG) + assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Deliver the block if version==2: @@ -641,15 +599,15 @@ class CompactBlocksTest(BitcoinTestFramework): num_to_request = random.randint(1, len(block.vtx)) msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) test_node.send_message(msg) - success = wait_until(lambda: test_node.last_blocktxn is not None, timeout=10) + success = wait_until(lambda: "blocktxn" in test_node.last_message, timeout=10) assert(success) [tx.calc_sha256() for tx in block.vtx] with mininode_lock: - assert_equal(test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16)) + assert_equal(test_node.last_message["blocktxn"].block_transactions.blockhash, int(block_hash, 16)) all_indices = msg.block_txn_request.to_absolute() for index in all_indices: - tx = test_node.last_blocktxn.block_transactions.transactions.pop(0) + tx = test_node.last_message["blocktxn"].block_transactions.transactions.pop(0) tx.calc_sha256() assert_equal(tx.sha256, block.vtx[index].sha256) if version == 1: @@ -658,7 +616,7 @@ class CompactBlocksTest(BitcoinTestFramework): else: # Check that the witness matches assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) - test_node.last_blocktxn = None + test_node.last_message.pop("blocktxn", None) current_height -= 1 # Next request should send a full block response, as we're past the @@ -666,13 +624,13 @@ class CompactBlocksTest(BitcoinTestFramework): block_hash = node.getblockhash(current_height) msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) with mininode_lock: - test_node.last_block = None - test_node.last_blocktxn = None + test_node.last_message.pop("block", None) + test_node.last_message.pop("blocktxn", None) test_node.send_and_ping(msg) with mininode_lock: - test_node.last_block.block.calc_sha256() - assert_equal(test_node.last_block.block.sha256, int(block_hash, 16)) - assert_equal(test_node.last_blocktxn, None) + test_node.last_message["block"].block.calc_sha256() + assert_equal(test_node.last_message["block"].block.sha256, int(block_hash, 16)) + assert "blocktxn" not in test_node.last_message def test_compactblocks_not_at_tip(self, node, test_node): # Test that requesting old compactblocks doesn't work. @@ -685,7 +643,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.clear_block_announcement() test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) - success = wait_until(lambda: test_node.last_cmpctblock is not None, timeout=30) + success = wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) assert(success) test_node.clear_block_announcement() @@ -693,13 +651,13 @@ class CompactBlocksTest(BitcoinTestFramework): wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() with mininode_lock: - test_node.last_block = None + test_node.last_message.pop("block", None) test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) - success = wait_until(lambda: test_node.last_block is not None, timeout=30) + success = wait_until(lambda: "block" in test_node.last_message, timeout=30) assert(success) with mininode_lock: - test_node.last_block.block.calc_sha256() - assert_equal(test_node.last_block.block.sha256, int(new_blocks[0], 16)) + test_node.last_message["block"].block.calc_sha256() + assert_equal(test_node.last_message["block"].block.sha256, int(new_blocks[0], 16)) # Generate an old compactblock, and verify that it's not accepted. cur_height = node.getblockcount() @@ -726,10 +684,10 @@ class CompactBlocksTest(BitcoinTestFramework): msg = msg_getblocktxn() msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) with mininode_lock: - test_node.last_blocktxn = None + test_node.last_message.pop("blocktxn", None) test_node.send_and_ping(msg) with mininode_lock: - assert(test_node.last_blocktxn is None) + assert "blocktxn" not in test_node.last_message def activate_segwit(self, node): node.generate(144*3) @@ -750,9 +708,9 @@ class CompactBlocksTest(BitcoinTestFramework): wait_until(lambda: l.received_block_announcement(), timeout=30) with mininode_lock: for l in listeners: - assert(l.last_cmpctblock is not None) - l.last_cmpctblock.header_and_shortids.header.calc_sha256() - assert_equal(l.last_cmpctblock.header_and_shortids.header.sha256, block.sha256) + assert "cmpctblock" in l.last_message + l.last_message["cmpctblock"].header_and_shortids.header.calc_sha256() + assert_equal(l.last_message["cmpctblock"].header_and_shortids.header.sha256, block.sha256) # Test that we don't get disconnected if we relay a compact block with valid header, # but invalid transactions. @@ -804,7 +762,7 @@ class CompactBlocksTest(BitcoinTestFramework): msg = msg_cmpctblock(cmpct_block.to_p2p()) peer.send_and_ping(msg) with mininode_lock: - assert(peer.last_getblocktxn is not None) + assert "getblocktxn" in peer.last_message return block, cmpct_block block, cmpct_block = announce_cmpct_block(node, stalling_peer) diff --git a/test/functional/p2p-feefilter.py b/test/functional/p2p-feefilter.py index 12539be950..dbccb633a5 100755 --- a/test/functional/p2p-feefilter.py +++ b/test/functional/p2p-feefilter.py @@ -22,8 +22,6 @@ def allInvsMatch(invsExpected, testnode): time.sleep(1) return False -# TestNode: bare-bones "peer". Used to track which invs are received from a node -# and to send the node feefilter messages. class TestNode(NodeConnCB): def __init__(self): super().__init__() @@ -38,10 +36,6 @@ class TestNode(NodeConnCB): with mininode_lock: self.txinvs = [] - def send_filter(self, feerate): - self.send_message(msg_feefilter(feerate)) - self.sync_with_ping() - class FeeFilterTest(BitcoinTestFramework): def __init__(self): @@ -49,14 +43,6 @@ class FeeFilterTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = False - def setup_network(self): - # Node1 will be used to generate txs which should be relayed from Node0 - # to our test node - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir)) - self.nodes.append(start_node(1, self.options.tmpdir)) - connect_nodes(self.nodes[0], 1) - def run_test(self): node1 = self.nodes[1] node0 = self.nodes[0] @@ -78,7 +64,7 @@ class FeeFilterTest(BitcoinTestFramework): test_node.clear_invs() # Set a filter of 15 sat/byte - test_node.send_filter(15000) + test_node.send_and_ping(msg_feefilter(15000)) # Test that txs are still being received (paying 20 sat/byte) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] @@ -103,7 +89,7 @@ class FeeFilterTest(BitcoinTestFramework): test_node.clear_invs() # Remove fee filter and check that txs are received again - test_node.send_filter(0) + test_node.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] assert(allInvsMatch(txids, test_node)) test_node.clear_invs() diff --git a/test/functional/p2p-leaktests.py b/test/functional/p2p-leaktests.py index 5853ec86f0..33b57ef33d 100755 --- a/test/functional/p2p-leaktests.py +++ b/test/functional/p2p-leaktests.py @@ -20,15 +20,8 @@ banscore = 10 class CLazyNode(NodeConnCB): def __init__(self): super().__init__() - self.connection = None self.unexpected_msg = False - self.connected = False - - def add_connection(self, conn): - self.connection = conn - - def send_message(self, message): - self.connection.send_message(message) + self.ever_connected = False def bad_message(self, message): self.unexpected_msg = True @@ -36,6 +29,7 @@ class CLazyNode(NodeConnCB): def on_open(self, conn): self.connected = True + self.ever_connected = True def on_version(self, conn, message): self.bad_message(message) def on_verack(self, conn, message): self.bad_message(message) @@ -63,9 +57,6 @@ class CLazyNode(NodeConnCB): # Node that never sends a version. We'll use this to send a bunch of messages # anyway, and eventually get disconnected. class CNodeNoVersionBan(CLazyNode): - def __init__(self): - super().__init__() - # send a bunch of veracks without sending a message. This should get us disconnected. # NOTE: implementation-specific check here. Remove if bitcoind ban behavior changes def on_open(self, conn): @@ -101,10 +92,7 @@ class P2PLeakTest(BitcoinTestFramework): def __init__(self): super().__init__() self.num_nodes = 1 - def setup_network(self): - extra_args = [['-banscore='+str(banscore)] - for i in range(self.num_nodes)] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.extra_args = [['-banscore='+str(banscore)]] def run_test(self): no_version_bannode = CNodeNoVersionBan() @@ -121,7 +109,9 @@ class P2PLeakTest(BitcoinTestFramework): NetworkThread().start() # Start up network handling in another thread - assert(wait_until(lambda: no_version_bannode.connected and no_version_idlenode.connected and no_verack_idlenode.version_received, timeout=10)) + assert wait_until(lambda: no_version_bannode.ever_connected, timeout=10) + assert wait_until(lambda: no_version_idlenode.ever_connected, timeout=10) + assert wait_until(lambda: no_verack_idlenode.version_received, timeout=10) # Mine a block and make sure that it's not sent to the connected nodes self.nodes[0].generate(1) @@ -130,7 +120,7 @@ class P2PLeakTest(BitcoinTestFramework): time.sleep(5) #This node should have been banned - assert(no_version_bannode.connection.state == "closed") + assert not no_version_bannode.connected [conn.disconnect_node() for conn in connections] diff --git a/test/functional/p2p-mempool.py b/test/functional/p2p-mempool.py index 5064ce74aa..34ef249eea 100755 --- a/test/functional/p2p-mempool.py +++ b/test/functional/p2p-mempool.py @@ -12,82 +12,24 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -class TestNode(NodeConnCB): - def __init__(self): - super().__init__() - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong() - self.block_receive_map = {} - - def add_connection(self, conn): - self.connection = conn - self.peer_disconnected = False - - def on_inv(self, conn, message): - pass - - # Track the last getdata message we receive (used in the test) - def on_getdata(self, conn, message): - self.last_getdata = message - - def on_block(self, conn, message): - message.block.calc_sha256() - try: - self.block_receive_map[message.block.sha256] += 1 - except KeyError as e: - self.block_receive_map[message.block.sha256] = 1 - - # Spin until verack message is received from the node. - # We use this to signal that our test can begin. This - # is called from the testing thread, so it needs to acquire - # the global lock. - def wait_for_verack(self): - def veracked(): - return self.verack_received - return wait_until(veracked, timeout=10) - - def wait_for_disconnect(self): - def disconnected(): - return self.peer_disconnected - return wait_until(disconnected, timeout=10) - - # Wrapper for the NodeConn's send_message function - def send_message(self, message): - self.connection.send_message(message) - - def on_pong(self, conn, message): - self.last_pong = message - - def on_close(self, conn): - self.peer_disconnected = True - - def send_mempool(self): - self.lastInv = [] - self.send_message(msg_mempool()) - class P2PMempoolTests(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True - self.num_nodes = 2 - - def setup_network(self): - # Start a node with maxuploadtarget of 200 MB (/24h) - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-peerbloomfilters=0"])) + self.num_nodes = 1 + self.extra_args = [["-peerbloomfilters=0"]] def run_test(self): #connect a mininode - aTestNode = TestNode() + aTestNode = NodeConnCB() node = NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], aTestNode) aTestNode.add_connection(node) NetworkThread().start() aTestNode.wait_for_verack() #request mempool - aTestNode.send_mempool() + aTestNode.send_message(msg_mempool()) aTestNode.wait_for_disconnect() #mininode must be disconnected at this point diff --git a/test/functional/p2p-segwit.py b/test/functional/p2p-segwit.py index cd7b788eb4..ab97596ba3 100755 --- a/test/functional/p2p-segwit.py +++ b/test/functional/p2p-segwit.py @@ -8,7 +8,7 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.script import * -from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, WITNESS_COMMITMENT_HEADER +from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, get_witness_script, WITNESS_COMMITMENT_HEADER from test_framework.key import CECKey, CPubKey import time import random @@ -35,79 +35,22 @@ def get_virtual_size(witness_block): class TestNode(NodeConnCB): def __init__(self): super().__init__() - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong(0) - self.sleep_time = 0.05 self.getdataset = set() - self.last_reject = None - - def add_connection(self, conn): - self.connection = conn - - # Wrapper for the NodeConn's send_message function - def send_message(self, message): - self.connection.send_message(message) - - def on_inv(self, conn, message): - self.last_inv = message - - def on_block(self, conn, message): - self.last_block = message.block - self.last_block.calc_sha256() def on_getdata(self, conn, message): for inv in message.inv: self.getdataset.add(inv.hash) - self.last_getdata = message - - def on_getheaders(self, conn, message): - self.last_getheaders = message - - def on_pong(self, conn, message): - self.last_pong = message - - def on_reject(self, conn, message): - self.last_reject = message - - # Syncing helpers - def sync(self, test_function, timeout=60): - while timeout > 0: - with mininode_lock: - if test_function(): - return - time.sleep(self.sleep_time) - timeout -= self.sleep_time - raise AssertionError("Sync failed to complete") - - def wait_for_block(self, blockhash, timeout=60): - test_function = lambda: self.last_block != None and self.last_block.sha256 == blockhash - self.sync(test_function, timeout) - return - - def wait_for_getdata(self, timeout=60): - test_function = lambda: self.last_getdata != None - self.sync(test_function, timeout) - - def wait_for_getheaders(self, timeout=60): - test_function = lambda: self.last_getheaders != None - self.sync(test_function, timeout) - - def wait_for_inv(self, expected_inv, timeout=60): - test_function = lambda: self.last_inv != expected_inv - self.sync(test_function, timeout) def announce_tx_and_wait_for_getdata(self, tx, timeout=60): with mininode_lock: - self.last_getdata = None + self.last_message.pop("getdata", None) self.send_message(msg_inv(inv=[CInv(1, tx.sha256)])) self.wait_for_getdata(timeout) - return def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): with mininode_lock: - self.last_getdata = None - self.last_getheaders = None + self.last_message.pop("getdata", None) + self.last_message.pop("getheaders", None) msg = msg_headers() msg.headers = [ CBlockHeader(block) ] if use_header: @@ -117,36 +60,25 @@ class TestNode(NodeConnCB): self.wait_for_getheaders() self.send_message(msg) self.wait_for_getdata() - return - - def announce_block(self, block, use_header): - with mininode_lock: - self.last_getdata = None - if use_header: - msg = msg_headers() - msg.headers = [ CBlockHeader(block) ] - self.send_message(msg) - else: - self.send_message(msg_inv(inv=[CInv(2, block.sha256)])) def request_block(self, blockhash, inv_type, timeout=60): with mininode_lock: - self.last_block = None + self.last_message.pop("block", None) self.send_message(msg_getdata(inv=[CInv(inv_type, blockhash)])) self.wait_for_block(blockhash, timeout) - return self.last_block + return self.last_message["block"].block def test_transaction_acceptance(self, tx, with_witness, accepted, reason=None): tx_message = msg_tx(tx) if with_witness: tx_message = msg_witness_tx(tx) self.send_message(tx_message) - self.sync_with_ping(60) + self.sync_with_ping() assert_equal(tx.hash in self.connection.rpc.getrawmempool(), accepted) if (reason != None and not accepted): # Check the rejection reason as well. with mininode_lock: - assert_equal(self.last_reject.reason, reason) + assert_equal(self.last_message["reject"].reason, reason) # Test whether a witness block had the correct effect on the tip def test_witness_block(self, block, accepted, with_witness=True): @@ -154,10 +86,9 @@ class TestNode(NodeConnCB): self.send_message(msg_witness_block(block)) else: self.send_message(msg_block(block)) - self.sync_with_ping(60) + self.sync_with_ping() assert_equal(self.connection.rpc.getbestblockhash() == block.hash, accepted) - # Used to keep track of anyone-can-spend outputs that we can use in the tests class UTXO(object): def __init__(self, sha256, n, nValue): @@ -183,17 +114,13 @@ class SegWitTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [["-whitelist=127.0.0.1"], ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0"], ["-whitelist=127.0.0.1", "-bip9params=segwit:0:0"]] def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-whitelist=127.0.0.1"])) - # Start a node for testing IsStandard rules. - self.nodes.append(start_node(1, self.options.tmpdir, ["-whitelist=127.0.0.1", "-acceptnonstdtxn=0"])) + self.setup_nodes() connect_nodes(self.nodes[0], 1) - - # Disable segwit's bip9 parameter to simulate upgrading after activation. - self.nodes.append(start_node(2, self.options.tmpdir, ["-whitelist=127.0.0.1", "-bip9params=segwit:0:0"])) connect_nodes(self.nodes[0], 2) + self.sync_all() ''' Helpers ''' # Build a block on top of node0's tip. @@ -228,7 +155,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block(nVersion=1) block.solve() self.test_node.send_message(msg_block(block)) - self.test_node.sync_with_ping(60) # make sure the block was processed + self.test_node.sync_with_ping() # make sure the block was processed txid = block.vtx[0].sha256 self.nodes[0].generate(99) # let the block mature @@ -244,7 +171,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(msg_tx(tx).serialize(), msg_witness_tx(tx).serialize()) self.test_node.send_message(msg_witness_tx(tx)) - self.test_node.sync_with_ping(60) # make sure the tx was processed + self.test_node.sync_with_ping() # make sure the tx was processed assert(tx.hash in self.nodes[0].getrawmempool()) # Save this transaction for later self.utxo.append(UTXO(tx.sha256, 0, 49*100000000)) @@ -279,12 +206,12 @@ class SegWitTest(BitcoinTestFramework): # TODO: fix synchronization so we can test reject reason # Right now, bitcoind delays sending reject messages for blocks # until the future, making synchronization here difficult. - #assert_equal(self.test_node.last_reject.reason, "unexpected-witness") + #assert_equal(self.test_node.last_message["reject"].reason, "unexpected-witness") # But it should not be permanently marked bad... # Resend without witness information. self.test_node.send_message(msg_block(block)) - self.test_node.sync_with_ping(60) + self.test_node.sync_with_ping() assert_equal(self.nodes[0].getbestblockhash(), block.hash) sync_blocks(self.nodes) @@ -893,7 +820,7 @@ class SegWitTest(BitcoinTestFramework): # Verify that if a peer doesn't set nServices to include NODE_WITNESS, # the getdata is just for the non-witness portion. self.old_node.announce_tx_and_wait_for_getdata(tx) - assert(self.old_node.last_getdata.inv[0].type == 1) + assert(self.old_node.last_message["getdata"].inv[0].type == 1) # Since we haven't delivered the tx yet, inv'ing the same tx from # a witness transaction ought not result in a getdata. @@ -989,9 +916,9 @@ class SegWitTest(BitcoinTestFramework): tx3.wit.vtxinwit[0].scriptWitness.stack = [ witness_program ] # Also check that old_node gets a tx announcement, even though this is # a witness transaction. - self.old_node.wait_for_inv(CInv(1, tx2.sha256)) # wait until tx2 was inv'ed + self.old_node.wait_for_inv([CInv(1, tx2.sha256)]) # wait until tx2 was inv'ed self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=True) - self.old_node.wait_for_inv(CInv(1, tx3.sha256)) + self.old_node.wait_for_inv([CInv(1, tx3.sha256)]) # Test that getrawtransaction returns correct witness information # hash, size, vsize @@ -1028,20 +955,20 @@ class SegWitTest(BitcoinTestFramework): block1.solve() self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) - assert(self.test_node.last_getdata.inv[0].type == blocktype) + assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) self.test_node.test_witness_block(block1, True) block2 = self.build_next_block(nVersion=4) block2.solve() self.test_node.announce_block_and_wait_for_getdata(block2, use_header=True) - assert(self.test_node.last_getdata.inv[0].type == blocktype) + assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) self.test_node.test_witness_block(block2, True) block3 = self.build_next_block(nVersion=(VB_TOP_BITS | (1<<15))) block3.solve() self.test_node.announce_block_and_wait_for_getdata(block3, use_header=True) - assert(self.test_node.last_getdata.inv[0].type == blocktype) + assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) self.test_node.test_witness_block(block3, True) # Check that we can getdata for witness blocks or regular blocks, @@ -1092,13 +1019,18 @@ class SegWitTest(BitcoinTestFramework): block4 = self.build_next_block(nVersion=4) block4.solve() self.old_node.getdataset = set() + # Blocks can be requested via direct-fetch (immediately upon processing the announcement) # or via parallel download (with an indeterminate delay from processing the announcement) # so to test that a block is NOT requested, we could guess a time period to sleep for, # and then check. We can avoid the sleep() by taking advantage of transaction getdata's # being processed after block getdata's, and announce a transaction as well, # and then check to see if that particular getdata has been received. - self.old_node.announce_block(block4, use_header=False) + # Since 0.14, inv's will only be responded to with a getheaders, so send a header + # to announce this block. + msg = msg_headers() + msg.headers = [ CBlockHeader(block4) ] + self.old_node.send_message(msg) self.old_node.announce_tx_and_wait_for_getdata(block4.vtx[0]) assert(block4.sha256 not in self.old_node.getdataset) @@ -1250,9 +1182,9 @@ class SegWitTest(BitcoinTestFramework): # Spending a higher version witness output is not allowed by policy, # even with fRequireStandard=false. self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=False) - self.test_node.sync_with_ping(60) + self.test_node.sync_with_ping() with mininode_lock: - assert(b"reserved for soft-fork upgrades" in self.test_node.last_reject.reason) + assert(b"reserved for soft-fork upgrades" in self.test_node.last_message["reject"].reason) # Building a block with the transaction must be valid, however. block = self.build_next_block() @@ -1380,7 +1312,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(NUM_TESTS): # Ping regularly to keep the connection alive if (not i % 100): - self.test_node.sync_with_ping(60) + self.test_node.sync_with_ping() # Choose random number of inputs to use. num_inputs = random.randint(1, 10) # Create a slight bias for producing more utxos @@ -1563,8 +1495,8 @@ class SegWitTest(BitcoinTestFramework): sync_blocks(self.nodes) # Restart with the new binary - stop_node(node, node_id) - self.nodes[node_id] = start_node(node_id, self.options.tmpdir) + self.stop_node(node_id) + self.nodes[node_id] = self.start_node(node_id, self.options.tmpdir) connect_nodes(self.nodes[0], node_id) sync_blocks(self.nodes) @@ -1721,15 +1653,10 @@ class SegWitTest(BitcoinTestFramework): assert('default_witness_commitment' in gbt_results) witness_commitment = gbt_results['default_witness_commitment'] - # TODO: this duplicates some code from blocktools.py, would be nice - # to refactor. # Check that default_witness_commitment is present. - block = CBlock() - witness_root = block.get_merkle_root([ser_uint256(0), ser_uint256(txid)]) - check_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(0))) - from test_framework.blocktools import WITNESS_COMMITMENT_HEADER - output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(check_commitment) - script = CScript([OP_RETURN, output_data]) + witness_root = CBlock.get_merkle_root([ser_uint256(0), + ser_uint256(txid)]) + script = get_witness_script(witness_root, 0) assert_equal(witness_commitment, bytes_to_hex_str(script)) # undo mocktime diff --git a/test/functional/p2p-timeouts.py b/test/functional/p2p-timeouts.py index de4edd6800..c3b29c215b 100755 --- a/test/functional/p2p-timeouts.py +++ b/test/functional/p2p-timeouts.py @@ -28,20 +28,9 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * class TestNode(NodeConnCB): - def __init__(self): - super().__init__() - self.connected = False - self.received_version = False - - def on_open(self, conn): - self.connected = True - - def on_close(self, conn): - self.connected = False - def on_version(self, conn, message): # Don't send a verack in response - self.received_version = True + pass class TimeoutsTest(BitcoinTestFramework): def __init__(self): @@ -49,12 +38,6 @@ class TimeoutsTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def setup_network(self): - self.nodes = [] - - # Start up node0 to be a version 1, pre-segwit node. - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - def run_test(self): # Setup the p2p connections and start up the network thread. self.no_verack_node = TestNode() # never send verack @@ -83,7 +66,7 @@ class TimeoutsTest(BitcoinTestFramework): sleep(30) - assert(self.no_verack_node.received_version) + assert "version" in self.no_verack_node.last_message assert(self.no_verack_node.connected) assert(self.no_version_node.connected) diff --git a/test/functional/p2p-versionbits-warning.py b/test/functional/p2p-versionbits-warning.py index da960e2d80..df7e8ce5c1 100755 --- a/test/functional/p2p-versionbits-warning.py +++ b/test/functional/p2p-versionbits-warning.py @@ -12,7 +12,6 @@ from test_framework.mininode import * from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * import re -import time from test_framework.blocktools import create_block, create_coinbase VB_PERIOD = 144 # versionbits period length for regtest @@ -24,28 +23,10 @@ WARN_UNKNOWN_RULES_MINED = "Unknown block versions being mined! It's possible un WARN_UNKNOWN_RULES_ACTIVE = "unknown new rules activated (versionbit {})".format(VB_UNKNOWN_BIT) VB_PATTERN = re.compile("^Warning.*versionbit") -# TestNode: bare-bones "peer". Used mostly as a conduit for a test to sending -# p2p messages to a node, generating the messages in the main testing logic. class TestNode(NodeConnCB): - def __init__(self): - super().__init__() - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong() - - def add_connection(self, conn): - self.connection = conn - def on_inv(self, conn, message): pass - # Wrapper for the NodeConn's send_message function - def send_message(self, message): - self.connection.send_message(message) - - def on_pong(self, conn, message): - self.last_pong = message - class VersionBitsWarningTest(BitcoinTestFramework): def __init__(self): super().__init__() @@ -58,7 +39,7 @@ class VersionBitsWarningTest(BitcoinTestFramework): with open(self.alert_filename, 'w', encoding='utf8') as _: pass self.extra_args = [["-alertnotify=echo %s >> \"" + self.alert_filename + "\""]] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + self.setup_nodes() # Send numblocks blocks via peer with nVersionToUse set. def send_blocks_with_version(self, peer, numblocks, nVersionToUse): @@ -127,22 +108,22 @@ class VersionBitsWarningTest(BitcoinTestFramework): # is cleared, and restart the node. This should move the versionbit state # to ACTIVE. self.nodes[0].generate(VB_PERIOD) - stop_nodes(self.nodes) + self.stop_nodes() # Empty out the alert file with open(self.alert_filename, 'w', encoding='utf8') as _: pass - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) # Connecting one block should be enough to generate an error. self.nodes[0].generate(1) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getinfo()["errors"]) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getmininginfo()["errors"]) assert(WARN_UNKNOWN_RULES_ACTIVE in self.nodes[0].getnetworkinfo()["warnings"]) - stop_nodes(self.nodes) + self.stop_nodes() self.test_versionbits_in_alert_file() # Test framework expects the node to still be running... - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) if __name__ == '__main__': VersionBitsWarningTest().main() diff --git a/test/functional/preciousblock.py b/test/functional/preciousblock.py index 30b0b5a301..04b41e76ba 100755 --- a/test/functional/preciousblock.py +++ b/test/functional/preciousblock.py @@ -41,7 +41,7 @@ class PreciousTest(BitcoinTestFramework): self.num_nodes = 3 def setup_network(self): - self.nodes = self.setup_nodes() + self.setup_nodes() def run_test(self): self.log.info("Ensure submitblock can in principle reorg to a competing chain") diff --git a/test/functional/prioritise_transaction.py b/test/functional/prioritise_transaction.py index 0b04ad17ab..9c3b3fd5d9 100755 --- a/test/functional/prioritise_transaction.py +++ b/test/functional/prioritise_transaction.py @@ -13,18 +13,13 @@ class PrioritiseTransactionTest(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True - self.num_nodes = 1 + self.num_nodes = 2 + self.extra_args = [["-printpriority=1"], ["-printpriority=1"]] + def run_test(self): self.txouts = gen_return_txouts() - - def setup_network(self): - self.nodes = [] - self.is_network_split = False - - self.nodes.append(start_node(0, self.options.tmpdir, ["-printpriority=1"])) self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] - def run_test(self): utxo_count = 90 utxos = create_confirmed_utxos(self.relayfee, self.nodes[0], utxo_count) base_fee = self.relayfee*100 # our transactions are smaller than 100kb @@ -120,5 +115,16 @@ class PrioritiseTransactionTest(BitcoinTestFramework): assert_equal(self.nodes[0].sendrawtransaction(tx_hex), tx_id) assert(tx_id in self.nodes[0].getrawmempool()) + # Test that calling prioritisetransaction is sufficient to trigger + # getblocktemplate to (eventually) return a new block. + mock_time = int(time.time()) + self.nodes[0].setmocktime(mock_time) + template = self.nodes[0].getblocktemplate() + self.nodes[0].prioritisetransaction(tx_id, -int(self.relayfee*COIN)) + self.nodes[0].setmocktime(mock_time+10) + new_template = self.nodes[0].getblocktemplate() + + assert(template != new_template) + if __name__ == '__main__': PrioritiseTransactionTest().main() diff --git a/test/functional/proxy_test.py b/test/functional/proxy_test.py index 748e3e69f6..ae6f843ddc 100755 --- a/test/functional/proxy_test.py +++ b/test/functional/proxy_test.py @@ -35,7 +35,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( PORT_MIN, PORT_RANGE, - start_nodes, assert_equal, ) from test_framework.netutil import test_ipv6_local @@ -90,7 +89,7 @@ class ProxyTest(BitcoinTestFramework): ] if self.have_ipv6: args[3] = ['-listen', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] - return start_nodes(self.num_nodes, self.options.tmpdir, extra_args=args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args=args) def node_test(self, node, proxies, auth, test_onion=True): rv = [] diff --git a/test/functional/pruning.py b/test/functional/pruning.py index cc84c8c085..4c3501ad77 100755 --- a/test/functional/pruning.py +++ b/test/functional/pruning.py @@ -32,31 +32,22 @@ class PruneTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 6 - # Cache for utxos, as the listunspent may take a long time later in the test - self.utxo_cache_0 = [] - self.utxo_cache_1 = [] - - def setup_network(self): - self.nodes = [] - self.is_network_split = False - - # Create nodes 0 and 1 to mine - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)) - self.nodes.append(start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900)) - - # Create node 2 to test pruning - self.nodes.append(start_node(2, self.options.tmpdir, ["-maxreceivebuffer=20000","-prune=550"], timewait=900)) - self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/" - + # Create nodes 0 and 1 to mine. + # Create node 2 to test pruning. + self.full_node_default_args = ["-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5", "-limitdescendantcount=100", "-limitdescendantsize=5000", "-limitancestorcount=100", "-limitancestorsize=5000" ] # Create nodes 3 and 4 to test manual pruning (they will be re-started with manual pruning later) - self.nodes.append(start_node(3, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900)) - self.nodes.append(start_node(4, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000"], timewait=900)) - # Create nodes 5 to test wallet in prune mode, but do not connect - self.nodes.append(start_node(5, self.options.tmpdir, ["-prune=550"])) + self.extra_args = [self.full_node_default_args, + self.full_node_default_args, + ["-maxreceivebuffer=20000", "-prune=550"], + ["-maxreceivebuffer=20000", "-blockmaxsize=999000"], + ["-maxreceivebuffer=20000", "-blockmaxsize=999000"], + ["-prune=550"]] - # Determine default relay fee - self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] + def setup_network(self): + self.setup_nodes() + + self.prunedir = self.options.tmpdir + "/node2/regtest/blocks/" connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[1], 2) @@ -107,12 +98,15 @@ class PruneTest(BitcoinTestFramework): # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine self.stop_node(0) - self.nodes[0]=start_node(0, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=900) + self.nodes[0]=self.start_node(0, self.options.tmpdir, self.full_node_default_args, timewait=900) # Mine 24 blocks in node 1 for i in range(24): if j == 0: mine_large_block(self.nodes[1], self.utxo_cache_1) else: + # Add node1's wallet transactions back to the mempool, to + # avoid the mined blocks from being too small. + self.nodes[1].resendwallettransactions() self.nodes[1].generate(1) #tx's already in mempool from previous disconnects # Reorg back with 25 block chain from node 0 @@ -132,7 +126,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) self.stop_node(1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) + self.nodes[1] = self.start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) height = self.nodes[1].getblockcount() self.log.info("Current block height: %d" % height) @@ -155,7 +149,7 @@ class PruneTest(BitcoinTestFramework): # Reboot node1 to clear those giant tx's from mempool self.stop_node(1) - self.nodes[1]=start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) + self.nodes[1] = self.start_node(1, self.options.tmpdir, ["-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=900) self.log.info("Generating new longer chain of 300 more blocks") self.nodes[1].generate(300) @@ -169,6 +163,11 @@ class PruneTest(BitcoinTestFramework): self.log.info("Usage possibly still high bc of stale blocks in block files: %d" % calc_usage(self.prunedir)) self.log.info("Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)") + + # Get node0's wallet transactions back in its mempool, to avoid the + # mined blocks from being too small. + self.nodes[0].resendwallettransactions() + for i in range(22): # This can be slow, so do this in multiple RPC calls to avoid # RPC timeouts. @@ -228,13 +227,13 @@ class PruneTest(BitcoinTestFramework): def manual_test(self, node_number, use_timestamp): # at this point, node has 995 blocks and has not yet run in prune mode - node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, timewait=900) + node = self.nodes[node_number] = self.start_node(node_number, self.options.tmpdir, timewait=900) assert_equal(node.getblockcount(), 995) assert_raises_jsonrpc(-1, "not in prune mode", node.pruneblockchain, 500) self.stop_node(node_number) # now re-start in manual pruning mode - node = self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-prune=1"], timewait=900) + node = self.nodes[node_number] = self.start_node(node_number, self.options.tmpdir, ["-prune=1"], timewait=900) assert_equal(node.getblockcount(), 995) def height(index): @@ -308,7 +307,7 @@ class PruneTest(BitcoinTestFramework): # stop node, start back up with auto-prune at 550MB, make sure still runs self.stop_node(node_number) - self.nodes[node_number] = start_node(node_number, self.options.tmpdir, ["-prune=550"], timewait=900) + self.nodes[node_number] = self.start_node(node_number, self.options.tmpdir, ["-prune=550"], timewait=900) self.log.info("Success") @@ -316,7 +315,7 @@ class PruneTest(BitcoinTestFramework): # check that the pruning node's wallet is still in good shape self.log.info("Stop and start pruning node to trigger wallet rescan") self.stop_node(2) - start_node(2, self.options.tmpdir, ["-prune=550"]) + self.start_node(2, self.options.tmpdir, ["-prune=550"]) self.log.info("Success") # check that wallet loads loads successfully when restarting a pruned node after IBD. @@ -326,12 +325,20 @@ class PruneTest(BitcoinTestFramework): nds = [self.nodes[0], self.nodes[5]] sync_blocks(nds, wait=5, timeout=300) self.stop_node(5) #stop and start to trigger rescan - start_node(5, self.options.tmpdir, ["-prune=550"]) + self.start_node(5, self.options.tmpdir, ["-prune=550"]) self.log.info("Success") def run_test(self): self.log.info("Warning! This test requires 4GB of disk space and takes over 30 mins (up to 2 hours)") self.log.info("Mining a big blockchain of 995 blocks") + + # Determine default relay fee + self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] + + # Cache for utxos, as the listunspent may take a long time later in the test + self.utxo_cache_0 = [] + self.utxo_cache_1 = [] + self.create_big_chain() # Chain diagram key: # * blocks on main chain diff --git a/test/functional/rawtransactions.py b/test/functional/rawtransactions.py index d4c684136c..35debf9cab 100755 --- a/test/functional/rawtransactions.py +++ b/test/functional/rawtransactions.py @@ -24,21 +24,9 @@ class RawTransactionsTest(BitcoinTestFramework): self.num_nodes = 3 def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - - #connect to a local machine for debugging - #url = "http://bitcoinrpc:DP6DvqZtqXarpeNWyN3LZTFchCCyCUuHwNF7E8pX99x1@%s:%d" % ('127.0.0.1', 18332) - #proxy = AuthServiceProxy(url) - #proxy.url = url # store URL on proxy for info - #self.nodes.append(proxy) - - connect_nodes_bi(self.nodes,0,1) - connect_nodes_bi(self.nodes,1,2) + super().setup_network() connect_nodes_bi(self.nodes,0,2) - self.is_network_split=False - self.sync_all() - def run_test(self): #prepare some coins for multiple *rawtransaction commands diff --git a/test/functional/receivedby.py b/test/functional/receivedby.py index 248bcdbd68..2cad6269ac 100755 --- a/test/functional/receivedby.py +++ b/test/functional/receivedby.py @@ -32,7 +32,7 @@ class ReceivedByTest(BitcoinTestFramework): def setup_nodes(self): #This test requires mocktime enable_mocktime() - return start_nodes(self.num_nodes, self.options.tmpdir) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir) def run_test(self): ''' diff --git a/test/functional/reindex.py b/test/functional/reindex.py index 0cebb0466f..b446baa04d 100755 --- a/test/functional/reindex.py +++ b/test/functional/reindex.py @@ -10,11 +10,7 @@ """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - start_nodes, - stop_nodes, - assert_equal, -) +from test_framework.util import assert_equal import time class ReindexTest(BitcoinTestFramework): @@ -24,15 +20,12 @@ class ReindexTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - def reindex(self, justchainstate=False): self.nodes[0].generate(3) blockcount = self.nodes[0].getblockcount() - stop_nodes(self.nodes) + self.stop_nodes() extra_args = [["-reindex-chainstate" if justchainstate else "-reindex", "-checkblockindex=1"]] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args) while self.nodes[0].getblockcount() < blockcount: time.sleep(0.1) assert_equal(self.nodes[0].getblockcount(), blockcount) diff --git a/test/functional/replace-by-fee.py b/test/functional/replace-by-fee.py index 163c304eba..e940ce535c 100755 --- a/test/functional/replace-by-fee.py +++ b/test/functional/replace-by-fee.py @@ -65,17 +65,12 @@ class ReplaceByFeeTest(BitcoinTestFramework): super().__init__() self.num_nodes = 1 self.setup_clean_chain = False - - def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", - "-whitelist=127.0.0.1", - "-limitancestorcount=50", - "-limitancestorsize=101", - "-limitdescendantcount=200", - "-limitdescendantsize=101" - ])) - self.is_network_split = False + self.extra_args= [["-maxorphantx=1000", + "-whitelist=127.0.0.1", + "-limitancestorcount=50", + "-limitancestorsize=101", + "-limitdescendantcount=200", + "-limitdescendantsize=101"]] def run_test(self): make_utxo(self.nodes[0], 1*COIN) diff --git a/test/functional/rest.py b/test/functional/rest.py index 776211d301..fbcceba0fa 100755 --- a/test/functional/rest.py +++ b/test/functional/rest.py @@ -49,12 +49,8 @@ class RESTTest (BitcoinTestFramework): self.num_nodes = 3 def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - connect_nodes_bi(self.nodes,0,1) - connect_nodes_bi(self.nodes,1,2) - connect_nodes_bi(self.nodes,0,2) - self.is_network_split=False - self.sync_all() + super().setup_network() + connect_nodes_bi(self.nodes, 0, 2) def run_test(self): url = urllib.parse.urlparse(self.nodes[0].url) diff --git a/test/functional/rpcbind_test.py b/test/functional/rpcbind_test.py index efc36481d1..5336cf2ec8 100755 --- a/test/functional/rpcbind_test.py +++ b/test/functional/rpcbind_test.py @@ -7,7 +7,7 @@ import socket import sys -from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_framework import BitcoinTestFramework, SkipTest from test_framework.util import * from test_framework.netutil import * @@ -36,10 +36,10 @@ class RPCBindTest(BitcoinTestFramework): if allow_ips: base_args += ['-rpcallowip=' + x for x in allow_ips] binds = ['-rpcbind='+addr for addr in addresses] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [base_args + binds], connect_to) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [base_args + binds], connect_to) pid = bitcoind_processes[0].pid assert_equal(set(get_bind_addrs(pid)), set(expected)) - stop_nodes(self.nodes) + self.stop_nodes() def run_allowip_test(self, allow_ips, rpchost, rpcport): ''' @@ -47,17 +47,16 @@ class RPCBindTest(BitcoinTestFramework): at a non-localhost IP. ''' base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [base_args]) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, [base_args]) # connect to node through non-loopback interface node = get_rpc_proxy(rpc_url(0, "%s:%d" % (rpchost, rpcport)), 0) node.getnetworkinfo() - stop_nodes(self.nodes) + self.stop_nodes() def run_test(self): # due to OS-specific network stats queries, this test works only on Linux if not sys.platform.startswith('linux'): - self.log.warning("This test can only be run on linux. Skipping test.") - sys.exit(self.TEST_EXIT_SKIPPED) + raise SkipTest("This test can only be run on linux.") # find the first non-loopback interface for testing non_loopback_ip = None for name,ip in all_interfaces(): @@ -65,15 +64,13 @@ class RPCBindTest(BitcoinTestFramework): non_loopback_ip = ip break if non_loopback_ip is None: - self.log.warning("This test requires at least one non-loopback IPv4 interface. Skipping test.") - sys.exit(self.TEST_EXIT_SKIPPED) + raise SkipTest("This test requires at least one non-loopback IPv4 interface.") try: s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) s.connect(("::1",1)) s.close except OSError: - self.log.warning("This test requires IPv6 support. Skipping test.") - sys.exit(self.TEST_EXIT_SKIPPED) + raise SkipTest("This test requires IPv6 support.") self.log.info("Using interface %s for testing" % non_loopback_ip) diff --git a/test/functional/rpcnamedargs.py b/test/functional/rpcnamedargs.py index f6175c8ca7..3b286000a1 100755 --- a/test/functional/rpcnamedargs.py +++ b/test/functional/rpcnamedargs.py @@ -8,7 +8,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_jsonrpc, - start_nodes, ) @@ -22,11 +21,6 @@ class NamedArgumentTest(BitcoinTestFramework): self.setup_clean_chain = False self.num_nodes = 1 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - self.is_network_split = False - self.sync_all() - def run_test(self): node = self.nodes[0] h = node.help(command='getinfo') diff --git a/test/functional/segwit.py b/test/functional/segwit.py index a1fffcb81a..ac95d66466 100755 --- a/test/functional/segwit.py +++ b/test/functional/segwit.py @@ -80,16 +80,13 @@ class SegWitTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 3 + self.extra_args = [["-walletprematurewitness", "-rpcserialversion=0"], + ["-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-rpcserialversion=1"], + ["-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness"]] def setup_network(self): - self.nodes = [] - self.nodes.append(start_node(0, self.options.tmpdir, ["-walletprematurewitness", "-rpcserialversion=0"])) - self.nodes.append(start_node(1, self.options.tmpdir, ["-blockversion=4", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness", "-rpcserialversion=1"])) - self.nodes.append(start_node(2, self.options.tmpdir, ["-blockversion=536870915", "-promiscuousmempoolflags=517", "-prematurewitness", "-walletprematurewitness"])) - connect_nodes(self.nodes[1], 0) - connect_nodes(self.nodes[2], 1) + super().setup_network() connect_nodes(self.nodes[0], 2) - self.is_network_split = False self.sync_all() def success_mine(self, node, txid, sign, redeem_script=""): diff --git a/test/functional/sendheaders.py b/test/functional/sendheaders.py index 1a7475ae84..44c357c6db 100755 --- a/test/functional/sendheaders.py +++ b/test/functional/sendheaders.py @@ -81,23 +81,17 @@ from test_framework.blocktools import create_block, create_coinbase direct_fetch_response_time = 0.05 -class BaseNode(NodeConnCB): +class TestNode(NodeConnCB): def __init__(self): super().__init__() - self.last_inv = None - self.last_headers = None - self.last_block = None - self.last_getdata = None self.block_announced = False - self.last_getheaders = None - self.disconnected = False self.last_blockhash_announced = None def clear_last_announcement(self): with mininode_lock: self.block_announced = False - self.last_inv = None - self.last_headers = None + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) # Request data for a list of block hashes def get_data(self, block_hashes): @@ -118,29 +112,17 @@ class BaseNode(NodeConnCB): self.connection.send_message(msg) def on_inv(self, conn, message): - self.last_inv = message self.block_announced = True self.last_blockhash_announced = message.inv[-1].hash def on_headers(self, conn, message): - self.last_headers = message if len(message.headers): self.block_announced = True message.headers[-1].calc_sha256() self.last_blockhash_announced = message.headers[-1].sha256 def on_block(self, conn, message): - self.last_block = message.block - self.last_block.calc_sha256() - - def on_getdata(self, conn, message): - self.last_getdata = message - - def on_getheaders(self, conn, message): - self.last_getheaders = message - - def on_close(self, conn): - self.disconnected = True + self.last_message["block"].calc_sha256() # Test whether the last announcement we received had the # right header or the right inv @@ -155,43 +137,27 @@ class BaseNode(NodeConnCB): success = True compare_inv = [] - if self.last_inv != None: - compare_inv = [x.hash for x in self.last_inv.inv] + if "inv" in self.last_message: + compare_inv = [x.hash for x in self.last_message["inv"].inv] if compare_inv != expect_inv: success = False hash_headers = [] - if self.last_headers != None: + if "headers" in self.last_message: # treat headers as a list of block hashes - hash_headers = [ x.sha256 for x in self.last_headers.headers ] + hash_headers = [ x.sha256 for x in self.last_message["headers"].headers ] if hash_headers != expect_headers: success = False - self.last_inv = None - self.last_headers = None + self.last_message.pop("inv", None) + self.last_message.pop("headers", None) return success - # Syncing helpers - def wait_for_block(self, blockhash, timeout=60): - test_function = lambda: self.last_block != None and self.last_block.sha256 == blockhash - assert(wait_until(test_function, timeout=timeout)) - return - - def wait_for_getheaders(self, timeout=60): - test_function = lambda: self.last_getheaders != None - assert(wait_until(test_function, timeout=timeout)) - return - def wait_for_getdata(self, hash_list, timeout=60): if hash_list == []: return - test_function = lambda: self.last_getdata != None and [x.hash for x in self.last_getdata.inv] == hash_list - assert(wait_until(test_function, timeout=timeout)) - return - - def wait_for_disconnect(self, timeout=60): - test_function = lambda: self.disconnected + test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list assert(wait_until(test_function, timeout=timeout)) return @@ -210,28 +176,12 @@ class BaseNode(NodeConnCB): getblocks_message.locator.vHave = locator self.send_message(getblocks_message) -# InvNode: This peer should only ever receive inv's, because it doesn't ever send a -# "sendheaders" message. -class InvNode(BaseNode): - def __init__(self): - BaseNode.__init__(self) - -# TestNode: This peer is the one we use for most of the testing. -class TestNode(BaseNode): - def __init__(self): - BaseNode.__init__(self) - class SendHeadersTest(BitcoinTestFramework): def __init__(self): super().__init__() self.setup_clean_chain = True self.num_nodes = 2 - def setup_network(self): - self.nodes = [] - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - connect_nodes(self.nodes[0], 1) - # mine count blocks and return the new tip def mine_blocks(self, count): # Clear out last block announcement from each p2p listener @@ -260,7 +210,7 @@ class SendHeadersTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections and start up the network thread. - inv_node = InvNode() + inv_node = TestNode() test_node = TestNode() self.p2p_connections = [inv_node, test_node] @@ -293,7 +243,7 @@ class SendHeadersTest(BitcoinTestFramework): if i == 0: # first request the block test_node.get_data([tip]) - test_node.wait_for_block(tip, timeout=5) + test_node.wait_for_block(tip) elif i == 1: # next try requesting header and block test_node.get_headers(locator=[old_tip], hashstop=tip) @@ -308,7 +258,7 @@ class SendHeadersTest(BitcoinTestFramework): new_block = create_block(tip, create_coinbase(height+1), block_time) new_block.solve() test_node.send_header_for_blocks([new_block]) - test_node.wait_for_getdata([new_block.sha256], timeout=5) + test_node.wait_for_getdata([new_block.sha256]) test_node.send_message(msg_block(new_block)) test_node.sync_with_ping() # make sure this block is processed inv_node.clear_last_announcement() @@ -347,18 +297,18 @@ class SendHeadersTest(BitcoinTestFramework): if j == 0: # Announce via inv test_node.send_block_inv(tip) - test_node.wait_for_getheaders(timeout=5) + test_node.wait_for_getheaders() # Should have received a getheaders now test_node.send_header_for_blocks(blocks) # Test that duplicate inv's won't result in duplicate # getdata requests, or duplicate headers announcements [ inv_node.send_block_inv(x.sha256) for x in blocks ] - test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=5) + test_node.wait_for_getdata([x.sha256 for x in blocks]) inv_node.sync_with_ping() else: # Announce via headers test_node.send_header_for_blocks(blocks) - test_node.wait_for_getdata([x.sha256 for x in blocks], timeout=5) + test_node.wait_for_getdata([x.sha256 for x in blocks]) # Test that duplicate headers won't result in duplicate # getdata requests (the check is further down) inv_node.send_header_for_blocks(blocks) @@ -368,8 +318,8 @@ class SendHeadersTest(BitcoinTestFramework): inv_node.sync_with_ping() # This block should not be announced to the inv node (since it also # broadcast it) - assert_equal(inv_node.last_inv, None) - assert_equal(inv_node.last_headers, None) + assert "inv" not in inv_node.last_message + assert "headers" not in inv_node.last_message tip = self.mine_blocks(1) assert_equal(inv_node.check_last_announcement(inv=[tip]), True) assert_equal(test_node.check_last_announcement(headers=[tip]), True) @@ -459,12 +409,12 @@ class SendHeadersTest(BitcoinTestFramework): inv_node.send_message(msg_block(blocks[-1])) inv_node.sync_with_ping() # Make sure blocks are processed - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks) test_node.sync_with_ping() # should not have received any getdata messages with mininode_lock: - assert_equal(test_node.last_getdata, None) + assert "getdata" not in test_node.last_message # This time, direct fetch should work blocks = [] @@ -498,11 +448,11 @@ class SendHeadersTest(BitcoinTestFramework): # Announcing one block on fork should not trigger direct fetch # (less work than tip) - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks[0:1]) test_node.sync_with_ping() with mininode_lock: - assert_equal(test_node.last_getdata, None) + assert "getdata" not in test_node.last_message # Announcing one more block on fork should trigger direct fetch for # both blocks (same work as tip) @@ -517,11 +467,11 @@ class SendHeadersTest(BitcoinTestFramework): test_node.wait_for_getdata([x.sha256 for x in blocks[2:16]], timeout=direct_fetch_response_time) # Announcing 1 more header should not trigger any response - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) test_node.send_header_for_blocks(blocks[18:19]) test_node.sync_with_ping() with mininode_lock: - assert_equal(test_node.last_getdata, None) + assert "getdata" not in test_node.last_message self.log.info("Part 4: success!") @@ -532,7 +482,7 @@ class SendHeadersTest(BitcoinTestFramework): # First we test that receipt of an unconnecting header doesn't prevent # chain sync. for i in range(10): - test_node.last_getdata = None + test_node.last_message.pop("getdata", None) blocks = [] # Create two more blocks. for j in range(2): @@ -543,9 +493,9 @@ class SendHeadersTest(BitcoinTestFramework): height += 1 # Send the header of the second block -> this won't connect. with mininode_lock: - test_node.last_getheaders = None + test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[1]]) - test_node.wait_for_getheaders(timeout=1) + test_node.wait_for_getheaders() test_node.send_header_for_blocks(blocks) test_node.wait_for_getdata([x.sha256 for x in blocks]) [ test_node.send_message(msg_block(x)) for x in blocks ] @@ -566,9 +516,9 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(1, MAX_UNCONNECTING_HEADERS): # Send a header that doesn't connect, check that we get a getheaders. with mininode_lock: - test_node.last_getheaders = None + test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i]]) - test_node.wait_for_getheaders(timeout=1) + test_node.wait_for_getheaders() # Next header will connect, should re-set our count: test_node.send_header_for_blocks([blocks[0]]) @@ -581,25 +531,21 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(5*MAX_UNCONNECTING_HEADERS - 1): # Send a header that doesn't connect, check that we get a getheaders. with mininode_lock: - test_node.last_getheaders = None + test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i%len(blocks)]]) - test_node.wait_for_getheaders(timeout=1) + test_node.wait_for_getheaders() # Eventually this stops working. - with mininode_lock: - self.last_getheaders = None test_node.send_header_for_blocks([blocks[-1]]) # Should get disconnected test_node.wait_for_disconnect() - with mininode_lock: - self.last_getheaders = True self.log.info("Part 5: success!") # Finally, check that the inv node never received a getdata request, # throughout the test - assert_equal(inv_node.last_getdata, None) + assert "getdata" not in inv_node.last_message if __name__ == '__main__': SendHeadersTest().main() diff --git a/test/functional/signmessages.py b/test/functional/signmessages.py index 91f5abef5d..42f6a9daaf 100755 --- a/test/functional/signmessages.py +++ b/test/functional/signmessages.py @@ -5,7 +5,6 @@ """Test RPC commands for signing and verifying messages.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * class SignMessagesTest(BitcoinTestFramework): @@ -14,10 +13,6 @@ class SignMessagesTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - self.is_network_split = False - def run_test(self): message = 'This is just a test message' diff --git a/test/functional/signrawtransactions.py b/test/functional/signrawtransactions.py index b24162ab97..437905e764 100755 --- a/test/functional/signrawtransactions.py +++ b/test/functional/signrawtransactions.py @@ -14,10 +14,6 @@ class SignRawTransactionsTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - self.is_network_split = False - def successful_signing_test(self): """Create and sign a valid raw transaction with one input. @@ -118,6 +114,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): # 5) Script verification errors have certain properties assert 'txid' in rawTxSigned['errors'][0] assert 'vout' in rawTxSigned['errors'][0] + assert 'witness' in rawTxSigned['errors'][0] assert 'scriptSig' in rawTxSigned['errors'][0] assert 'sequence' in rawTxSigned['errors'][0] assert 'error' in rawTxSigned['errors'][0] @@ -127,6 +124,32 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert_equal(rawTxSigned['errors'][0]['vout'], inputs[1]['vout']) assert_equal(rawTxSigned['errors'][1]['txid'], inputs[2]['txid']) assert_equal(rawTxSigned['errors'][1]['vout'], inputs[2]['vout']) + assert not rawTxSigned['errors'][0]['witness'] + + # Now test signing failure for transaction with input witnesses + p2wpkh_raw_tx = "01000000000102fff7f7881a8099afa6940d42d1e7f6362bec38171ea3edf433541db4e4ad969f00000000494830450221008b9d1dc26ba6a9cb62127b02742fa9d754cd3bebf337f7a55d114c8e5cdd30be022040529b194ba3f9281a99f2b1c0a19c0489bc22ede944ccf4ecbab4cc618ef3ed01eeffffffef51e1b804cc89d182d279655c3aa89e815b1b309fe287d9b2b55d57b90ec68a0100000000ffffffff02202cb206000000001976a9148280b37df378db99f66f85c95a783a76ac7a6d5988ac9093510d000000001976a9143bde42dbee7e4dbe6a21b2d50ce2f0167faa815988ac000247304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee0121025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee635711000000" + + rawTxSigned = self.nodes[0].signrawtransaction(p2wpkh_raw_tx) + + # 7) The transaction has no complete set of signatures + assert 'complete' in rawTxSigned + assert_equal(rawTxSigned['complete'], False) + + # 8) Two script verification errors occurred + assert 'errors' in rawTxSigned + assert_equal(len(rawTxSigned['errors']), 2) + + # 9) Script verification errors have certain properties + assert 'txid' in rawTxSigned['errors'][0] + assert 'vout' in rawTxSigned['errors'][0] + assert 'witness' in rawTxSigned['errors'][0] + assert 'scriptSig' in rawTxSigned['errors'][0] + assert 'sequence' in rawTxSigned['errors'][0] + assert 'error' in rawTxSigned['errors'][0] + + # Non-empty witness checked here + assert_equal(rawTxSigned['errors'][1]['witness'], ["304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee01", "025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee6357"]) + assert not rawTxSigned['errors'][0]['witness'] def run_test(self): self.successful_signing_test() diff --git a/test/functional/smartfees.py b/test/functional/smartfees.py index b7f4b86e7b..482c77863f 100755 --- a/test/functional/smartfees.py +++ b/test/functional/smartfees.py @@ -116,8 +116,8 @@ def check_estimates(node, fees_seen, max_invalid, print_estimates = True): for i,e in enumerate(all_estimates): # estimate is for i+1 if e >= 0: valid_estimate = True - # estimatesmartfee should return the same result - assert_equal(node.estimatesmartfee(i+1)["feerate"], e) + if i >= 13: # for n>=14 estimatesmartfee(n/2) should be at least as high as estimatefee(n) + assert(node.estimatesmartfee((i+1)//2)["feerate"] > float(e) - delta) else: invalid_estimates += 1 @@ -155,7 +155,7 @@ class EstimateFeeTest(BitcoinTestFramework): """ self.nodes = [] # Use node0 to mine blocks for input splitting - self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", + self.nodes.append(self.start_node(0, self.options.tmpdir, ["-maxorphantx=1000", "-whitelist=127.0.0.1"])) self.log.info("This test is time consuming, please be patient") @@ -191,7 +191,7 @@ class EstimateFeeTest(BitcoinTestFramework): # Node1 mines small blocks but that are bigger than the expected transaction rate. # NOTE: the CreateNewBlock code starts counting block size at 1,000 bytes, # (17k is room enough for 110 or so transactions) - self.nodes.append(start_node(1, self.options.tmpdir, + self.nodes.append(self.start_node(1, self.options.tmpdir, ["-blockmaxsize=17000", "-maxorphantx=1000"])) connect_nodes(self.nodes[1], 0) @@ -199,11 +199,10 @@ class EstimateFeeTest(BitcoinTestFramework): # produces too small blocks (room for only 55 or so transactions) node2args = ["-blockmaxsize=8000", "-maxorphantx=1000"] - self.nodes.append(start_node(2, self.options.tmpdir, node2args)) + self.nodes.append(self.start_node(2, self.options.tmpdir, node2args)) connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[2], 1) - self.is_network_split = False self.sync_all() def transact_and_mine(self, numblocks, mining_node): diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 9ab3094b06..dfcc524313 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -42,6 +42,7 @@ import decimal import json import logging import socket +import time try: import urllib.parse as urlparse except ImportError: @@ -163,6 +164,7 @@ class AuthServiceProxy(object): return self._request('POST', self.__url.path, postdata.encode('utf-8')) def _get_response(self): + req_start_time = time.time() try: http_response = self.__conn.getresponse() except socket.timeout as e: @@ -183,8 +185,9 @@ class AuthServiceProxy(object): responsedata = http_response.read().decode('utf8') response = json.loads(responsedata, parse_float=decimal.Decimal) + elapsed = time.time() - req_start_time if "error" in response and response["error"] is None: - log.debug("<-%s- %s"%(response["id"], json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) + log.debug("<-%s- [%.6f] %s"%(response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii))) else: - log.debug("<-- "+responsedata) + log.debug("<-- [%.6f] %s"%(elapsed,responsedata)) return response diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 2c9a0857df..5dcf516dc6 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -25,6 +25,13 @@ def create_block(hashprev, coinbase, nTime=None): # From BIP141 WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed" + +def get_witness_script(witness_root, witness_nonce): + witness_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(witness_nonce))) + output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(witness_commitment) + return CScript([OP_RETURN, output_data]) + + # According to BIP141, blocks with witness rules active must commit to the # hash of all in-block transactions including witness. def add_witness_commitment(block, nonce=0): @@ -32,14 +39,12 @@ def add_witness_commitment(block, nonce=0): # transactions, with witnesses. witness_nonce = nonce witness_root = block.calc_witness_merkle_root() - witness_commitment = uint256_from_str(hash256(ser_uint256(witness_root)+ser_uint256(witness_nonce))) # witness_nonce should go to coinbase witness. block.vtx[0].wit.vtxinwit = [CTxInWitness()] block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(witness_nonce)] # witness commitment is the last OP_RETURN output in coinbase - output_data = WITNESS_COMMITMENT_HEADER + ser_uint256(witness_commitment) - block.vtx[0].vout.append(CTxOut(0, CScript([OP_RETURN, output_data]))) + block.vtx[0].vout.append(CTxOut(0, get_witness_script(witness_root, witness_nonce))) block.vtx[0].rehash() block.hashMerkleRoot = block.calc_merkle_root() block.rehash() diff --git a/test/functional/test_framework/comptool.py b/test/functional/test_framework/comptool.py index 25c18bda82..9f062865a3 100755 --- a/test/functional/test_framework/comptool.py +++ b/test/functional/test_framework/comptool.py @@ -192,9 +192,7 @@ class TestManager(object): return wait_until(disconnected, timeout=10) def wait_for_verack(self): - def veracked(): - return all(node.verack_received for node in self.test_nodes) - return wait_until(veracked, timeout=10) + return all(node.wait_for_verack() for node in self.test_nodes) def wait_for_pings(self, counter): def received_pongs(): diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index d57d46f2ff..fb3ed1473a 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -20,21 +20,22 @@ msg_block, msg_tx, msg_headers, etc.: ser_*, deser_*: functions that handle serialization/deserialization """ -import struct -import socket import asyncore -import time -import sys -import random -from .util import hex_str_to_bytes, bytes_to_hex_str -from io import BytesIO from codecs import encode +from collections import defaultdict +import copy import hashlib -from threading import RLock -from threading import Thread +from io import BytesIO import logging -import copy +import random +import socket +import struct +import sys +import time +from threading import RLock, Thread + from test_framework.siphash import siphash256 +from test_framework.util import hex_str_to_bytes, bytes_to_hex_str BIP0031_VERSION = 60000 MY_VERSION = 70014 # past bip-31 for ping/pong @@ -610,7 +611,8 @@ class CBlock(CBlockHeader): return r # Calculate the merkle root given a vector of transaction hashes - def get_merkle_root(self, hashes): + @classmethod + def get_merkle_root(cls, hashes): while len(hashes) > 1: newhashes = [] for i in range(0, len(hashes), 2): @@ -1356,6 +1358,8 @@ class msg_reject(object): # Helper function def wait_until(predicate, *, attempts=float('inf'), timeout=float('inf')): + if attempts == float('inf') and timeout == float('inf'): + timeout = 60 attempt = 0 elapsed = 0 @@ -1465,30 +1469,57 @@ class msg_witness_blocktxn(msg_blocktxn): r += self.block_transactions.serialize(with_witness=True) return r -# This is what a callback should look like for NodeConn -# Reimplement the on_* functions to provide handling for events class NodeConnCB(object): + """Callback and helper functions for P2P connection to a bitcoind node. + + Individual testcases should subclass this and override the on_* methods + if they want to alter message handling behaviour. + """ + def __init__(self): - self.verack_received = False + # Track whether we have a P2P connection open to the node + self.connected = False + self.connection = None + + # Track number of messages of each type received and the most recent + # message of each type + self.message_count = defaultdict(int) + self.last_message = {} + + # A count of the number of ping messages we've sent to the node + self.ping_counter = 1 + # deliver_sleep_time is helpful for debugging race conditions in p2p # tests; it causes message delivery to sleep for the specified time # before acquiring the global lock and delivering the next message. self.deliver_sleep_time = None + # Remember the services our peer has advertised self.peer_services = None - self.connection = None - self.ping_counter = 1 - self.last_pong = msg_pong() + + # Message receiving methods def deliver(self, conn, message): + """Receive message and dispatch message to appropriate callback. + + We keep a count of how many of each message type has been received + and the most recent message of each type. + + Optionally waits for deliver_sleep_time before dispatching message. + """ + deliver_sleep = self.get_deliver_sleep_time() if deliver_sleep is not None: time.sleep(deliver_sleep) with mininode_lock: try: - getattr(self, 'on_' + message.command.decode('ascii'))(conn, message) + command = message.command.decode('ascii') + self.message_count[command] += 1 + self.last_message[command] = message + getattr(self, 'on_' + command)(conn, message) except: - logger.exception("ERROR delivering %s" % repr(message)) + print("ERROR delivering %s (%s)" % (repr(message), + sys.exc_info()[0])) def set_deliver_sleep_time(self, value): with mininode_lock: @@ -1498,14 +1529,20 @@ class NodeConnCB(object): with mininode_lock: return self.deliver_sleep_time - # Callbacks which can be overridden by subclasses - ################################################# + # Callback methods. Can be overridden by subclasses in individual test + # cases to provide custom message handling behaviour. + + def on_open(self, conn): + self.connected = True + + def on_close(self, conn): + self.connected = False + self.connection = None def on_addr(self, conn, message): pass def on_alert(self, conn, message): pass def on_block(self, conn, message): pass def on_blocktxn(self, conn, message): pass - def on_close(self, conn): pass def on_cmpctblock(self, conn, message): pass def on_feefilter(self, conn, message): pass def on_getaddr(self, conn, message): pass @@ -1515,7 +1552,7 @@ class NodeConnCB(object): def on_getheaders(self, conn, message): pass def on_headers(self, conn, message): pass def on_mempool(self, conn): pass - def on_open(self, conn): pass + def on_pong(self, conn, message): pass def on_reject(self, conn, message): pass def on_sendcmpct(self, conn, message): pass def on_sendheaders(self, conn, message): pass @@ -1533,9 +1570,6 @@ class NodeConnCB(object): if conn.ver_send > BIP0031_VERSION: conn.send_message(msg_pong(message.nonce)) - def on_pong(self, conn, message): - self.last_pong = message - def on_verack(self, conn, message): conn.ver_recv = conn.ver_send self.verack_received = True @@ -1548,15 +1582,49 @@ class NodeConnCB(object): conn.ver_recv = conn.ver_send conn.nServices = message.nServices - # Helper functions - ################## + # Connection helper methods def add_connection(self, conn): self.connection = conn - # Wrapper for the NodeConn's send_message function + def wait_for_disconnect(self, timeout=60): + test_function = lambda: not self.connected + assert wait_until(test_function, timeout=timeout) + + # Message receiving helper methods + + def wait_for_block(self, blockhash, timeout=60): + test_function = lambda: self.last_message.get("block") and self.last_message["block"].block.rehash() == blockhash + assert wait_until(test_function, timeout=timeout) + + def wait_for_getdata(self, timeout=60): + test_function = lambda: self.last_message.get("getdata") + assert wait_until(test_function, timeout=timeout) + + def wait_for_getheaders(self, timeout=60): + test_function = lambda: self.last_message.get("getheaders") + assert wait_until(test_function, timeout=timeout) + + def wait_for_inv(self, expected_inv, timeout=60): + """Waits for an INV message and checks that the first inv object in the message was as expected.""" + if len(expected_inv) > 1: + raise NotImplementedError("wait_for_inv() will only verify the first inv object") + test_function = lambda: self.last_message.get("inv") and \ + self.last_message["inv"].inv[0].type == expected_inv[0].type and \ + self.last_message["inv"].inv[0].hash == expected_inv[0].hash + assert wait_until(test_function, timeout=timeout) + + def wait_for_verack(self, timeout=60): + test_function = lambda: self.message_count["verack"] + assert wait_until(test_function, timeout=timeout) + + # Message sending helper functions + def send_message(self, message): - self.connection.send_message(message) + if self.connection: + self.connection.send_message(message) + else: + logger.error("Cannot send message. No connection to node!") def send_and_ping(self, message): self.send_message(message) @@ -1564,27 +1632,11 @@ class NodeConnCB(object): # Sync up with the node def sync_with_ping(self, timeout=60): - def received_pong(): - return (self.last_pong.nonce == self.ping_counter) self.send_message(msg_ping(nonce=self.ping_counter)) - success = wait_until(received_pong, timeout=timeout) - if not success: - logger.error("sync_with_ping failed!") - raise AssertionError("sync_with_ping failed!") + test_function = lambda: self.last_message.get("pong") and self.last_message["pong"].nonce == self.ping_counter + assert wait_until(test_function, timeout=timeout) self.ping_counter += 1 - - return success - - # Spin until verack message is received from the node. - # Tests may want to use this as a signal that the test can begin. - # This can be called from the testing thread, so it needs to acquire the - # global lock. - def wait_for_verack(self): - while True: - with mininode_lock: - if self.verack_received: - return - time.sleep(0.05) + return True # The actual NodeConn class # This class provides an interface for a p2p connection to a specified node diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 473b7c14a9..67abf35687 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -5,103 +5,103 @@ """Base class for RPC testing.""" from collections import deque +from enum import Enum import logging import optparse import os -import sys import shutil +import subprocess +import sys import tempfile import time from .util import ( - initialize_chain, - start_nodes, + PortSeed, + MAX_NODES, + bitcoind_processes, + check_json_precision, connect_nodes_bi, + disable_mocktime, + disconnect_nodes, + enable_coverage, + enable_mocktime, + get_mocktime, + get_rpc_proxy, + initialize_datadir, + log_filename, + p2p_port, + rpc_url, + set_node_times, + _start_node, + _start_nodes, + _stop_node, + _stop_nodes, sync_blocks, sync_mempools, - stop_nodes, - stop_node, - enable_coverage, - check_json_precision, - initialize_chain_clean, - PortSeed, + wait_for_bitcoind_start, ) from .authproxy import JSONRPCException +class TestStatus(Enum): + PASSED = 1 + FAILED = 2 + SKIPPED = 3 + +TEST_EXIT_PASSED = 0 +TEST_EXIT_FAILED = 1 +TEST_EXIT_SKIPPED = 77 + class BitcoinTestFramework(object): + """Base class for a bitcoin test script. + + Individual bitcoin test scripts should subclass this class and override the following methods: + + - __init__() + - add_options() + - setup_chain() + - setup_network() + - run_test() - TEST_EXIT_PASSED = 0 - TEST_EXIT_FAILED = 1 - TEST_EXIT_SKIPPED = 77 + The main() method should not be overridden. + This class also contains various public and private helper methods.""" + + # Methods to override in subclass test scripts. def __init__(self): self.num_nodes = 4 self.setup_clean_chain = False self.nodes = None - def run_test(self): - raise NotImplementedError - def add_options(self, parser): pass def setup_chain(self): self.log.info("Initializing test directory "+self.options.tmpdir) if self.setup_clean_chain: - initialize_chain_clean(self.options.tmpdir, self.num_nodes) + self._initialize_chain_clean(self.options.tmpdir, self.num_nodes) else: - initialize_chain(self.options.tmpdir, self.num_nodes, self.options.cachedir) + self._initialize_chain(self.options.tmpdir, self.num_nodes, self.options.cachedir) - def stop_node(self, num_node): - stop_node(self.nodes[num_node], num_node) - - def setup_nodes(self): - return start_nodes(self.num_nodes, self.options.tmpdir) - - def setup_network(self, split = False): - self.nodes = self.setup_nodes() + def setup_network(self): + self.setup_nodes() # Connect the nodes as a "chain". This allows us # to split the network between nodes 1 and 2 to get # two halves that can work on competing chains. - - # If we joined network halves, connect the nodes from the joint - # on outward. This ensures that chains are properly reorganised. - if not split: - connect_nodes_bi(self.nodes, 1, 2) - sync_blocks(self.nodes[1:3]) - sync_mempools(self.nodes[1:3]) - - connect_nodes_bi(self.nodes, 0, 1) - connect_nodes_bi(self.nodes, 2, 3) - self.is_network_split = split + for i in range(self.num_nodes - 1): + connect_nodes_bi(self.nodes, i, i + 1) self.sync_all() - def split_network(self): - """ - Split the network of four nodes into nodes 0/1 and 2/3. - """ - assert not self.is_network_split - stop_nodes(self.nodes) - self.setup_network(True) - - def sync_all(self): - if self.is_network_split: - sync_blocks(self.nodes[:2]) - sync_blocks(self.nodes[2:]) - sync_mempools(self.nodes[:2]) - sync_mempools(self.nodes[2:]) - else: - sync_blocks(self.nodes) - sync_mempools(self.nodes) + def setup_nodes(self): + extra_args = None + if hasattr(self, "extra_args"): + extra_args = self.extra_args + self.nodes = _start_nodes(self.num_nodes, self.options.tmpdir, extra_args) - def join_network(self): - """ - Join the (previously split) network halves together. - """ - assert self.is_network_split - stop_nodes(self.nodes) - self.setup_network(False) + def run_test(self): + raise NotImplementedError + + # Main function. This should not be overridden by the subclass test scripts. def main(self): @@ -114,8 +114,7 @@ class BitcoinTestFramework(object): help="Source directory containing bitcoind/bitcoin-cli (default: %default)") parser.add_option("--cachedir", dest="cachedir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../cache"), help="Directory for caching pregenerated datadirs") - parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"), - help="Root directory for datadirs") + parser.add_option("--tmpdir", dest="tmpdir", help="Root directory for datadirs") parser.add_option("-l", "--loglevel", dest="loglevel", default="INFO", help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. Note that logs at all levels are always written to the test_framework.log file in the temporary test directory.") parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true", @@ -124,12 +123,11 @@ class BitcoinTestFramework(object): help="The seed to use for assigning port numbers (default: current process id)") parser.add_option("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") + parser.add_option("--configfile", dest="configfile", + help="Location of the test framework config file") self.add_options(parser) (self.options, self.args) = parser.parse_args() - # backup dir variable for removal at cleanup - self.options.root, self.options.tmpdir = self.options.tmpdir, self.options.tmpdir + '/' + str(self.options.port_seed) - if self.options.coveragedir: enable_coverage(self.options.coveragedir) @@ -140,18 +138,24 @@ class BitcoinTestFramework(object): check_json_precision() # Set up temp directory and start logging - os.makedirs(self.options.tmpdir, exist_ok=False) + if self.options.tmpdir: + os.makedirs(self.options.tmpdir, exist_ok=False) + else: + self.options.tmpdir = tempfile.mkdtemp(prefix="test") self._start_logging() - success = False + success = TestStatus.FAILED try: self.setup_chain() self.setup_network() self.run_test() - success = True + success = TestStatus.PASSED except JSONRPCException as e: self.log.exception("JSONRPC error") + except SkipTest as e: + self.log.warning("Test Skipped: %s" % e.message) + success = TestStatus.SKIPPED except AssertionError as e: self.log.exception("Assertion failed") except KeyError as e: @@ -163,15 +167,14 @@ class BitcoinTestFramework(object): if not self.options.noshutdown: self.log.info("Stopping nodes") - stop_nodes(self.nodes) + if self.nodes: + self.stop_nodes() else: self.log.info("Note: bitcoinds were not stopped and may still be running") - if not self.options.nocleanup and not self.options.noshutdown and success: + if not self.options.nocleanup and not self.options.noshutdown and success != TestStatus.FAILED: self.log.info("Cleaning up") shutil.rmtree(self.options.tmpdir) - if not os.listdir(self.options.root): - os.rmdir(self.options.root) else: self.log.warning("Not cleaning up dir %s" % self.options.tmpdir) if os.getenv("PYTHON_DEBUG", ""): @@ -189,13 +192,56 @@ class BitcoinTestFramework(object): except OSError: print("Opening file %s failed." % fn) traceback.print_exc() - if success: + + if success == TestStatus.PASSED: self.log.info("Tests successful") - sys.exit(self.TEST_EXIT_PASSED) + sys.exit(TEST_EXIT_PASSED) + elif success == TestStatus.SKIPPED: + self.log.info("Test skipped") + sys.exit(TEST_EXIT_SKIPPED) else: self.log.error("Test failed. Test logging available at %s/test_framework.log", self.options.tmpdir) logging.shutdown() - sys.exit(self.TEST_EXIT_FAILED) + sys.exit(TEST_EXIT_FAILED) + + # Public helper methods. These can be accessed by the subclass test scripts. + + def start_node(self, i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): + return _start_node(i, dirname, extra_args, rpchost, timewait, binary, stderr) + + def start_nodes(self, num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): + return _start_nodes(num_nodes, dirname, extra_args, rpchost, timewait, binary) + + def stop_node(self, num_node): + _stop_node(self.nodes[num_node], num_node) + + def stop_nodes(self): + _stop_nodes(self.nodes) + + def split_network(self): + """ + Split the network of four nodes into nodes 0/1 and 2/3. + """ + disconnect_nodes(self.nodes[1], 2) + disconnect_nodes(self.nodes[2], 1) + self.sync_all([self.nodes[:2], self.nodes[2:]]) + + def join_network(self): + """ + Join the (previously split) network halves together. + """ + connect_nodes_bi(self.nodes, 1, 2) + self.sync_all() + + def sync_all(self, node_groups=None): + if not node_groups: + node_groups = [self.nodes] + + for group in node_groups: + sync_blocks(group) + sync_mempools(group) + + # Private helper methods. These should not be accessed by the subclass test scripts. def _start_logging(self): # Add logger and logging handlers @@ -225,12 +271,99 @@ class BitcoinTestFramework(object): rpc_handler.setLevel(logging.DEBUG) rpc_logger.addHandler(rpc_handler) + def _initialize_chain(self, test_dir, num_nodes, cachedir): + """Initialize a pre-mined blockchain for use by the test. + + Create a cache of a 200-block-long chain (with wallet) for MAX_NODES + Afterward, create num_nodes copies from the cache.""" + + assert num_nodes <= MAX_NODES + create_cache = False + for i in range(MAX_NODES): + if not os.path.isdir(os.path.join(cachedir, 'node' + str(i))): + create_cache = True + break + + if create_cache: + self.log.debug("Creating data directories from cached datadir") + + # find and delete old cache directories if any exist + for i in range(MAX_NODES): + if os.path.isdir(os.path.join(cachedir, "node" + str(i))): + shutil.rmtree(os.path.join(cachedir, "node" + str(i))) + + # Create cache directories, run bitcoinds: + for i in range(MAX_NODES): + datadir = initialize_datadir(cachedir, i) + args = [os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir=" + datadir, "-discover=0"] + if i > 0: + args.append("-connect=127.0.0.1:" + str(p2p_port(0))) + bitcoind_processes[i] = subprocess.Popen(args) + self.log.debug("initialize_chain: bitcoind started, waiting for RPC to come up") + wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i) + self.log.debug("initialize_chain: RPC successfully started") + + self.nodes = [] + for i in range(MAX_NODES): + try: + self.nodes.append(get_rpc_proxy(rpc_url(i), i)) + except: + self.log.exception("Error connecting to node %d" % i) + sys.exit(1) + + # Create a 200-block-long chain; each of the 4 first nodes + # gets 25 mature blocks and 25 immature. + # Note: To preserve compatibility with older versions of + # initialize_chain, only 4 nodes will generate coins. + # + # blocks are created with timestamps 10 minutes apart + # starting from 2010 minutes in the past + enable_mocktime() + block_time = get_mocktime() - (201 * 10 * 60) + for i in range(2): + for peer in range(4): + for j in range(25): + set_node_times(self.nodes, block_time) + self.nodes[peer].generate(1) + block_time += 10 * 60 + # Must sync before next peer starts generating blocks + sync_blocks(self.nodes) + + # Shut them down, and clean up cache directories: + self.stop_nodes() + self.nodes = [] + disable_mocktime() + for i in range(MAX_NODES): + os.remove(log_filename(cachedir, i, "debug.log")) + os.remove(log_filename(cachedir, i, "db.log")) + os.remove(log_filename(cachedir, i, "peers.dat")) + os.remove(log_filename(cachedir, i, "fee_estimates.dat")) + + for i in range(num_nodes): + from_dir = os.path.join(cachedir, "node" + str(i)) + to_dir = os.path.join(test_dir, "node" + str(i)) + shutil.copytree(from_dir, to_dir) + initialize_datadir(test_dir, i) # Overwrite port/rpcport in bitcoin.conf + + def _initialize_chain_clean(self, test_dir, num_nodes): + """Initialize empty blockchain for use by the test. + + Create an empty blockchain and num_nodes wallets. + Useful if a test case wants complete control over initialization.""" + for i in range(num_nodes): + initialize_datadir(test_dir, i) + # Test framework for doing p2p comparison testing, which sets up some bitcoind # binaries: # 1 binary: test binary # 2 binaries: 1 test binary, 1 ref binary # n>2 binaries: 1 test binary, n-1 ref binaries +class SkipTest(Exception): + """This exception is raised to skip a test""" + def __init__(self, message): + self.message = message + class ComparisonTestFramework(BitcoinTestFramework): def __init__(self): @@ -247,7 +380,7 @@ class ComparisonTestFramework(BitcoinTestFramework): help="bitcoind binary to use for reference nodes (if any)") def setup_network(self): - self.nodes = start_nodes( + self.nodes = self.start_nodes( self.num_nodes, self.options.tmpdir, extra_args=[['-whitelist=127.0.0.1']] * self.num_nodes, binary=[self.options.testbinary] + diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 899b0b5a1b..2b0f32c2b6 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -226,96 +226,16 @@ def wait_for_bitcoind_start(process, url, i): raise # unknown JSON RPC exception time.sleep(0.25) -def initialize_chain(test_dir, num_nodes, cachedir): - """ - Create a cache of a 200-block-long chain (with wallet) for MAX_NODES - Afterward, create num_nodes copies from the cache - """ - assert num_nodes <= MAX_NODES - create_cache = False - for i in range(MAX_NODES): - if not os.path.isdir(os.path.join(cachedir, 'node'+str(i))): - create_cache = True - break - - if create_cache: - logger.debug("Creating data directories from cached datadir") - - #find and delete old cache directories if any exist - for i in range(MAX_NODES): - if os.path.isdir(os.path.join(cachedir,"node"+str(i))): - shutil.rmtree(os.path.join(cachedir,"node"+str(i))) - - # Create cache directories, run bitcoinds: - for i in range(MAX_NODES): - datadir=initialize_datadir(cachedir, i) - args = [ os.getenv("BITCOIND", "bitcoind"), "-server", "-keypool=1", "-datadir="+datadir, "-discover=0" ] - if i > 0: - args.append("-connect=127.0.0.1:"+str(p2p_port(0))) - bitcoind_processes[i] = subprocess.Popen(args) - logger.debug("initialize_chain: bitcoind started, waiting for RPC to come up") - wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i) - logger.debug("initialize_chain: RPC successfully started") - - rpcs = [] - for i in range(MAX_NODES): - try: - rpcs.append(get_rpc_proxy(rpc_url(i), i)) - except: - sys.stderr.write("Error connecting to "+url+"\n") - sys.exit(1) - - # Create a 200-block-long chain; each of the 4 first nodes - # gets 25 mature blocks and 25 immature. - # Note: To preserve compatibility with older versions of - # initialize_chain, only 4 nodes will generate coins. - # - # blocks are created with timestamps 10 minutes apart - # starting from 2010 minutes in the past - enable_mocktime() - block_time = get_mocktime() - (201 * 10 * 60) - for i in range(2): - for peer in range(4): - for j in range(25): - set_node_times(rpcs, block_time) - rpcs[peer].generate(1) - block_time += 10*60 - # Must sync before next peer starts generating blocks - sync_blocks(rpcs) - - # Shut them down, and clean up cache directories: - stop_nodes(rpcs) - disable_mocktime() - for i in range(MAX_NODES): - os.remove(log_filename(cachedir, i, "debug.log")) - os.remove(log_filename(cachedir, i, "db.log")) - os.remove(log_filename(cachedir, i, "peers.dat")) - os.remove(log_filename(cachedir, i, "fee_estimates.dat")) - - for i in range(num_nodes): - from_dir = os.path.join(cachedir, "node"+str(i)) - to_dir = os.path.join(test_dir, "node"+str(i)) - shutil.copytree(from_dir, to_dir) - initialize_datadir(test_dir, i) # Overwrite port/rpcport in bitcoin.conf - -def initialize_chain_clean(test_dir, num_nodes): - """ - Create an empty blockchain and num_nodes wallets. - Useful if a test case wants complete control over initialization. - """ - for i in range(num_nodes): - datadir=initialize_datadir(test_dir, i) +def _start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): + """Start a bitcoind and return RPC connection to it + This function should only be called from within test_framework, not by individual test scripts.""" -def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): - """ - Start a bitcoind and return RPC connection to it - """ datadir = os.path.join(dirname, "node"+str(i)) if binary is None: binary = os.getenv("BITCOIND", "bitcoind") - args = [binary, "-datadir=" + datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(get_mocktime())] + args = [binary, "-datadir=" + datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-logtimemicros", "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", "-mocktime=" + str(get_mocktime()), "-uacomment=testnode%d" % i] if extra_args is not None: args.extend(extra_args) bitcoind_processes[i] = subprocess.Popen(args, stderr=stderr) logger.debug("initialize_chain: bitcoind started, waiting for RPC to come up") @@ -332,8 +252,8 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=None): with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: try: - node = start_node(i, dirname, extra_args, stderr=log_stderr) - stop_node(node, i) + node = _start_node(i, dirname, extra_args, stderr=log_stderr) + _stop_node(node, i) except Exception as e: assert 'bitcoind exited' in str(e) #node must have shutdown if expected_msg is not None: @@ -348,25 +268,32 @@ def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=Non assert_msg = "bitcoind should have exited with expected error " + expected_msg raise AssertionError(assert_msg) -def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): - """ - Start multiple bitcoinds, return RPC connections to them - """ +def _start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): + """Start multiple bitcoinds, return RPC connections to them + + This function should only be called from within test_framework, not by individual test scripts.""" + if extra_args is None: extra_args = [ None for _ in range(num_nodes) ] if binary is None: binary = [ None for _ in range(num_nodes) ] + assert_equal(len(extra_args), num_nodes) + assert_equal(len(binary), num_nodes) rpcs = [] try: for i in range(num_nodes): - rpcs.append(start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i])) + rpcs.append(_start_node(i, dirname, extra_args[i], rpchost, timewait=timewait, binary=binary[i])) except: # If one node failed to start, stop the others - stop_nodes(rpcs) + _stop_nodes(rpcs) raise return rpcs def log_filename(dirname, n_node, logname): return os.path.join(dirname, "node"+str(n_node), "regtest", logname) -def stop_node(node, i): +def _stop_node(node, i): + """Stop a bitcoind test node + + This function should only be called from within test_framework, not by individual test scripts.""" + logger.debug("Stopping node %d" % i) try: node.stop() @@ -376,15 +303,30 @@ def stop_node(node, i): assert_equal(return_code, 0) del bitcoind_processes[i] -def stop_nodes(nodes): +def _stop_nodes(nodes): + """Stop multiple bitcoind test nodes + + This function should only be called from within test_framework, not by individual test scripts.""" + for i, node in enumerate(nodes): - stop_node(node, i) + _stop_node(node, i) assert not bitcoind_processes.values() # All connections must be gone now def set_node_times(nodes, t): for node in nodes: node.setmocktime(t) +def disconnect_nodes(from_connection, node_num): + for peer_id in [peer['id'] for peer in from_connection.getpeerinfo() if "testnode%d" % node_num in peer['subver']]: + from_connection.disconnectnode(nodeid=peer_id) + + for _ in range(50): + if [peer['id'] for peer in from_connection.getpeerinfo() if "testnode%d" % node_num in peer['subver']] == []: + break + time.sleep(0.1) + else: + raise AssertionError("timed out waiting for disconnect") + def connect_nodes(from_connection, node_num): ip_port = "127.0.0.1:"+str(p2p_port(node_num)) from_connection.addnode(ip_port, "onetry") diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index bb12328ec1..b2aee7c739 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -16,6 +16,7 @@ For a description of arguments recognized by test scripts, see import argparse import configparser +import datetime import os import time import shutil @@ -27,9 +28,17 @@ import logging # Formatting. Default colors to empty strings. BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "") -TICK = "✓ " -CROSS = "✖ " -CIRCLE = "○ " +try: + # Make sure python thinks it can write unicode to its stdout + "\u2713".encode("utf_8").decode(sys.stdout.encoding) + TICK = "✓ " + CROSS = "✖ " + CIRCLE = "○ " +except UnicodeDecodeError: + TICK = "P " + CROSS = "x " + CIRCLE = "o " + if os.name == 'posix': # primitive formatting on supported # terminal via ANSI escape sequences: @@ -69,6 +78,7 @@ BASE_SCRIPTS= [ 'rawtransactions.py', 'reindex.py', # vv Tests less than 30s vv + "zmq_test.py", 'mempool_resurrect_test.py', 'txn_doublespend.py --mineblock', 'txn_clone.py', @@ -76,11 +86,12 @@ BASE_SCRIPTS= [ 'rest.py', 'mempool_spendcoinbase.py', 'mempool_reorg.py', + 'mempool_persist.py', 'httpbasics.py', 'multi_rpc.py', 'proxy_test.py', 'signrawtransactions.py', - 'nodehandling.py', + 'disconnect_ban.py', 'decodescript.py', 'blockchain.py', 'disablewallet.py', @@ -102,11 +113,6 @@ BASE_SCRIPTS= [ 'p2p-leaktests.py', ] -ZMQ_SCRIPTS = [ - # ZMQ test can only be run if bitcoin was built with zmq-enabled. - # call test_runner.py with -nozmq to explicitly exclude these tests. - 'zmq_test.py'] - EXTENDED_SCRIPTS = [ # These tests are not run by the travis build process. # Longest test should go first, to favor running tests in parallel @@ -135,12 +141,12 @@ EXTENDED_SCRIPTS = [ 'txn_clone.py --mineblock', 'forknotify.py', 'invalidateblock.py', - 'maxblocksinflight.py', 'p2p-acceptblock.py', 'replace-by-fee.py', ] -ALL_SCRIPTS = BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS +# Place EXTENDED_SCRIPTS first since it has the 3 longest running tests +ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS NON_SCRIPTS = [ # These are python files that live in the functional tests directory, but are not test scripts. @@ -158,31 +164,40 @@ def main(): Help text and arguments for individual test script:''', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface') - parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude. Do not include the .py extension in the name.') + parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') + parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') - parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests') + parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") args, unknown_args = parser.parse_known_args() - # Create a set to store arguments and create the passon string - tests = set(arg for arg in unknown_args if arg[:2] != "--") + # args to be passed on always start with two dashes; tests are the remaining unknown args + tests = [arg for arg in unknown_args if arg[:2] != "--"] passon_args = [arg for arg in unknown_args if arg[:2] == "--"] # Read config generated by configure. config = configparser.ConfigParser() - config.read_file(open(os.path.dirname(__file__) + "/config.ini")) + configfile = os.path.abspath(os.path.dirname(__file__)) + "/config.ini" + config.read_file(open(configfile)) + + passon_args.append("--configfile=%s" % configfile) # Set up logging logging_level = logging.INFO if args.quiet else logging.DEBUG logging.basicConfig(format='%(message)s', level=logging_level) + # Create base test directory + tmpdir = "%s/bitcoin_test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) + os.makedirs(tmpdir) + + logging.debug("Temporary test directory at %s" % tmpdir) + enable_wallet = config["components"].getboolean("ENABLE_WALLET") enable_utils = config["components"].getboolean("ENABLE_UTILS") enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") - enable_zmq = config["components"].getboolean("ENABLE_ZMQ") and not args.nozmq if config["environment"]["EXEEXT"] == ".exe" and not args.force: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 @@ -195,38 +210,34 @@ def main(): print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make") sys.exit(0) - # python3-zmq may not be installed. Handle this gracefully and with some helpful info - if enable_zmq: - try: - import zmq - except ImportError: - print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests." - "To run zmq tests, see dependency info in /test/README.md.") - raise - # Build list of tests if tests: # Individual tests have been specified. Run specified tests that exist # in the ALL_SCRIPTS list. Accept the name with or without .py extension. - test_list = [t for t in ALL_SCRIPTS if - (t in tests or re.sub(".py$", "", t) in tests)] + tests = [re.sub("\.py$", "", t) + ".py" for t in tests] + test_list = [] + for t in tests: + if t in ALL_SCRIPTS: + test_list.append(t) + else: + print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], t)) else: - # No individual tests have been specified. Run base tests, and - # optionally ZMQ tests and extended tests. + # No individual tests have been specified. + # Run all base tests, and optionally run extended tests. test_list = BASE_SCRIPTS - if enable_zmq: - test_list += ZMQ_SCRIPTS if args.extended: - test_list += EXTENDED_SCRIPTS - # TODO: BASE_SCRIPTS and EXTENDED_SCRIPTS are sorted by runtime - # (for parallel running efficiency). This combined list will is no - # longer sorted. + # place the EXTENDED_SCRIPTS first since the three longest ones + # are there and the list is shorter + test_list = EXTENDED_SCRIPTS + test_list # Remove the test cases that the user has explicitly asked to exclude. if args.exclude: - for exclude_test in args.exclude.split(','): - if exclude_test + ".py" in test_list: - test_list.remove(exclude_test + ".py") + tests_excl = [re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',')] + for exclude_test in tests_excl: + if exclude_test in test_list: + test_list.remove(exclude_test) + else: + print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], exclude_test)) if not test_list: print("No valid test scripts specified. Check that your test is in one " @@ -241,9 +252,23 @@ def main(): check_script_list(config["environment"]["SRCDIR"]) - run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], args.jobs, args.coverage, passon_args) + if not args.keepcache: + shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True) + + run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args) -def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=False, args=[]): +def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]): + # Warn if bitcoind is already running (unix only) + try: + if subprocess.check_output(["pidof", "bitcoind"]) is not None: + print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0])) + except (OSError, subprocess.SubprocessError): + pass + + # Warn if there is a cache directory + cache_dir = "%s/test/cache" % build_dir + if os.path.isdir(cache_dir): + print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir)) #Set env vars if "BITCOIND" not in os.environ: @@ -252,7 +277,7 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal tests_dir = src_dir + '/test/functional/' flags = ["--srcdir={}/src".format(build_dir)] + args - flags.append("--cachedir=%s/test/cache" % build_dir) + flags.append("--cachedir=%s" % cache_dir) if enable_coverage: coverage = RPCCoverage() @@ -263,10 +288,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal if len(test_list) > 1 and jobs > 1: # Populate cache - subprocess.check_output([tests_dir + 'create_cache.py'] + flags) + subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir]) #Run Tests - job_queue = TestHandler(jobs, tests_dir, test_list, flags) + job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags) time0 = time.time() test_results = [] @@ -293,6 +318,10 @@ def run_tests(test_list, src_dir, build_dir, exeext, jobs=1, enable_coverage=Fal logging.debug("Cleaning up coverage data") coverage.cleanup() + # Clear up the temp directory if all subdirectories are gone + if not os.listdir(tmpdir): + os.rmdir(tmpdir) + all_passed = all(map(lambda test_result: test_result.was_successful, test_results)) sys.exit(not all_passed) @@ -320,10 +349,11 @@ class TestHandler: Trigger the testscrips passed in via the list. """ - def __init__(self, num_tests_parallel, tests_dir, test_list=None, flags=None): + def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None): assert(num_tests_parallel >= 1) self.num_jobs = num_tests_parallel self.tests_dir = tests_dir + self.tmpdir = tmpdir self.test_list = test_list self.flags = flags self.num_running = 0 @@ -338,13 +368,15 @@ class TestHandler: # Add tests self.num_running += 1 t = self.test_list.pop(0) - port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)] + portseed = len(self.test_list) + self.portseed_offset + portseed_arg = ["--portseed={}".format(portseed)] log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) test_argv = t.split() + tmpdir = ["--tmpdir=%s/%s_%s" % (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)] self.jobs.append((t, time.time(), - subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + port_seed, + subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir, universal_newlines=True, stdout=log_stdout, stderr=log_stderr), @@ -407,9 +439,10 @@ def check_script_list(src_dir): python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"]) missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS))) if len(missed_tests) != 0: - print("The following scripts are not being run:" + str(missed_tests)) - print("Check the test lists in test_runner.py") - sys.exit(1) + print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests))) + if os.getenv('TRAVIS') == 'true': + # On travis this warning is an error to prevent merging incomplete commits into master + sys.exit(1) class RPCCoverage(object): """ diff --git a/test/functional/txn_clone.py b/test/functional/txn_clone.py index 7a3b8d3474..9b81af96cf 100755 --- a/test/functional/txn_clone.py +++ b/test/functional/txn_clone.py @@ -20,7 +20,9 @@ class TxnMallTest(BitcoinTestFramework): def setup_network(self): # Start with split network: - return super(TxnMallTest, self).setup_network(True) + super(TxnMallTest, self).setup_network() + disconnect_nodes(self.nodes[1], 2) + disconnect_nodes(self.nodes[2], 1) def run_test(self): # All nodes should start with 1,250 BTC: diff --git a/test/functional/txn_doublespend.py b/test/functional/txn_doublespend.py index 5b12cf4c29..1bd3b3271c 100755 --- a/test/functional/txn_doublespend.py +++ b/test/functional/txn_doublespend.py @@ -20,7 +20,9 @@ class TxnMallTest(BitcoinTestFramework): def setup_network(self): # Start with split network: - return super(TxnMallTest, self).setup_network(True) + super().setup_network() + disconnect_nodes(self.nodes[1], 2) + disconnect_nodes(self.nodes[2], 1) def run_test(self): # All nodes should start with 1,250 BTC: diff --git a/test/functional/wallet-accounts.py b/test/functional/wallet-accounts.py index ea12d4ec22..158aa9ae89 100755 --- a/test/functional/wallet-accounts.py +++ b/test/functional/wallet-accounts.py @@ -7,16 +7,14 @@ RPCs tested are: - getaccountaddress - getaddressesbyaccount + - listaddressgroupings - setaccount - sendfrom (with account arguments) - move (with account arguments) """ from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - start_nodes, - assert_equal, -) +from test_framework.util import assert_equal class WalletAccountsTest(BitcoinTestFramework): @@ -24,22 +22,59 @@ class WalletAccountsTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 1 - self.node_args = [[]] + self.extra_args = [[]] - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.node_args) - self.is_network_split = False - - def run_test (self): + def run_test(self): node = self.nodes[0] # Check that there's no UTXO on any of the nodes assert_equal(len(node.listunspent()), 0) - + + # Note each time we call generate, all generated coins go into + # the same address, so we call twice to get two addresses w/50 each + node.generate(1) node.generate(101) - - assert_equal(node.getbalance(), 50) - - accounts = ["a","b","c","d","e"] + assert_equal(node.getbalance(), 100) + + # there should be 2 address groups + # each with 1 address with a balance of 50 Bitcoins + address_groups = node.listaddressgroupings() + assert_equal(len(address_groups), 2) + # the addresses aren't linked now, but will be after we send to the + # common address + linked_addresses = set() + for address_group in address_groups: + assert_equal(len(address_group), 1) + assert_equal(len(address_group[0]), 2) + assert_equal(address_group[0][1], 50) + linked_addresses.add(address_group[0][0]) + + # send 50 from each address to a third address not in this wallet + # There's some fee that will come back to us when the miner reward + # matures. + common_address = "msf4WtN1YQKXvNtvdFYt9JBnUD2FB41kjr" + txid = node.sendmany( + fromaccount="", + amounts={common_address: 100}, + subtractfeefrom=[common_address], + minconf=1, + ) + tx_details = node.gettransaction(txid) + fee = -tx_details['details'][0]['fee'] + # there should be 1 address group, with the previously + # unlinked addresses now linked (they both have 0 balance) + address_groups = node.listaddressgroupings() + assert_equal(len(address_groups), 1) + assert_equal(len(address_groups[0]), 2) + assert_equal(set([a[0] for a in address_groups[0]]), linked_addresses) + assert_equal([a[1] for a in address_groups[0]], [0, 0]) + + node.generate(1) + + # we want to reset so that the "" account has what's expected. + # otherwise we're off by exactly the fee amount as that's mined + # and matures in the next 100 blocks + node.sendfrom("", common_address, fee) + accounts = ["a", "b", "c", "d", "e"] amount_to_send = 1.0 account_addresses = dict() for account in accounts: @@ -56,7 +91,7 @@ class WalletAccountsTest(BitcoinTestFramework): for i in range(len(accounts)): from_account = accounts[i] - to_account = accounts[(i+1)%len(accounts)] + to_account = accounts[(i+1) % len(accounts)] to_address = account_addresses[to_account] node.sendfrom(from_account, to_address, amount_to_send) @@ -67,7 +102,7 @@ class WalletAccountsTest(BitcoinTestFramework): assert(address != account_addresses[account]) assert_equal(node.getreceivedbyaccount(account), 2) node.move(account, "", node.getbalance(account)) - + node.generate(101) expected_account_balances = {"": 5200} @@ -97,4 +132,4 @@ class WalletAccountsTest(BitcoinTestFramework): assert_equal(node.getbalance(account), 50) if __name__ == '__main__': - WalletAccountsTest().main () + WalletAccountsTest().main() diff --git a/test/functional/wallet-dump.py b/test/functional/wallet-dump.py index 352add28fb..9cb32d4650 100755 --- a/test/functional/wallet-dump.py +++ b/test/functional/wallet-dump.py @@ -7,7 +7,7 @@ import os from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import (start_nodes, start_node, assert_equal, bitcoind_processes) +from test_framework.util import (assert_equal, bitcoind_processes) def read_dump(file_name, addrs, hd_master_addr_old): @@ -68,7 +68,7 @@ class WalletDumpTest(BitcoinTestFramework): # longer than the default 30 seconds due to an expensive # CWallet::TopUpKeyPool call, and the encryptwallet RPC made later in # the test often takes even longer. - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args, timewait=60) + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args, timewait=60) def run_test (self): tmpdir = self.options.tmpdir @@ -96,7 +96,7 @@ class WalletDumpTest(BitcoinTestFramework): #encrypt wallet, restart, unlock and dump self.nodes[0].encryptwallet('test') bitcoind_processes[0].wait() - self.nodes[0] = start_node(0, self.options.tmpdir, self.extra_args[0]) + self.nodes[0] = self.start_node(0, self.options.tmpdir, self.extra_args[0]) self.nodes[0].walletpassphrase('test', 10) # Should be a no-op: self.nodes[0].keypoolrefill() diff --git a/test/functional/wallet-hd.py b/test/functional/wallet-hd.py index 64a6c92782..e7ec72a248 100755 --- a/test/functional/wallet-hd.py +++ b/test/functional/wallet-hd.py @@ -6,8 +6,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - start_nodes, - start_node, assert_equal, connect_nodes_bi, assert_start_raises_init_error @@ -22,12 +20,7 @@ class WalletHDTest(BitcoinTestFramework): super().__init__() self.setup_clean_chain = True self.num_nodes = 2 - self.node_args = [['-usehd=0'], ['-usehd=1', '-keypool=0']] - - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.node_args) - self.is_network_split = False - connect_nodes_bi(self.nodes, 0, 1) + self.extra_args = [['-usehd=0'], ['-usehd=1', '-keypool=0']] def run_test (self): tmpdir = self.options.tmpdir @@ -35,7 +28,7 @@ class WalletHDTest(BitcoinTestFramework): # Make sure can't switch off usehd after wallet creation self.stop_node(1) assert_start_raises_init_error(1, self.options.tmpdir, ['-usehd=0'], 'already existing HD wallet') - self.nodes[1] = start_node(1, self.options.tmpdir, self.node_args[1]) + self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1]) connect_nodes_bi(self.nodes, 0, 1) # Make sure we use hd, keep masterkeyid @@ -44,7 +37,7 @@ class WalletHDTest(BitcoinTestFramework): # create an internal key change_addr = self.nodes[1].getrawchangeaddress() - change_addrV= self.nodes[1].validateaddress(change_addr); + change_addrV= self.nodes[1].validateaddress(change_addr) assert_equal(change_addrV["hdkeypath"], "m/0'/1'/0'") #first internal child key # Import a non-HD private key in the HD wallet @@ -72,7 +65,7 @@ class WalletHDTest(BitcoinTestFramework): # create an internal key (again) change_addr = self.nodes[1].getrawchangeaddress() - change_addrV= self.nodes[1].validateaddress(change_addr); + change_addrV= self.nodes[1].validateaddress(change_addr) assert_equal(change_addrV["hdkeypath"], "m/0'/1'/1'") #second internal child key self.sync_all() @@ -82,7 +75,7 @@ class WalletHDTest(BitcoinTestFramework): self.stop_node(1) os.remove(self.options.tmpdir + "/node1/regtest/wallet.dat") shutil.copyfile(tmpdir + "/hd.bak", tmpdir + "/node1/regtest/wallet.dat") - self.nodes[1] = start_node(1, self.options.tmpdir, self.node_args[1]) + self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1]) #connect_nodes_bi(self.nodes, 0, 1) # Assert that derivation is deterministic @@ -96,13 +89,13 @@ class WalletHDTest(BitcoinTestFramework): # Needs rescan self.stop_node(1) - self.nodes[1] = start_node(1, self.options.tmpdir, self.node_args[1] + ['-rescan']) + self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1] + ['-rescan']) #connect_nodes_bi(self.nodes, 0, 1) assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1) # send a tx and make sure its using the internal chain for the changeoutput txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) - outs = self.nodes[1].decoderawtransaction(self.nodes[1].gettransaction(txid)['hex'])['vout']; + outs = self.nodes[1].decoderawtransaction(self.nodes[1].gettransaction(txid)['hex'])['vout'] keypath = "" for out in outs: if out['value'] != 1: diff --git a/test/functional/wallet.py b/test/functional/wallet.py index 80f74fa108..3e3e8fcddb 100755 --- a/test/functional/wallet.py +++ b/test/functional/wallet.py @@ -6,7 +6,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -class WalletTest (BitcoinTestFramework): +class WalletTest(BitcoinTestFramework): def check_fee_amount(self, curr_balance, balance_with_fee, fee_per_byte, tx_size): """Return curr_balance after asserting the fee was in range""" @@ -20,15 +20,14 @@ class WalletTest (BitcoinTestFramework): self.num_nodes = 4 self.extra_args = [['-usehd={:d}'.format(i%2==0)] for i in range(4)] - def setup_network(self, split=False): - self.nodes = start_nodes(3, self.options.tmpdir, self.extra_args[:3]) + def setup_network(self): + self.nodes = self.start_nodes(3, self.options.tmpdir, self.extra_args[:3]) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) - self.is_network_split=False self.sync_all() - def run_test (self): + def run_test(self): # Check that there's no UTXO on none of the nodes assert_equal(len(self.nodes[0].listunspent()), 0) @@ -52,14 +51,38 @@ class WalletTest (BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), 0) # Check that only first and second nodes have UTXOs - assert_equal(len(self.nodes[0].listunspent()), 1) + utxos = self.nodes[0].listunspent() + assert_equal(len(utxos), 1) assert_equal(len(self.nodes[1].listunspent()), 1) assert_equal(len(self.nodes[2].listunspent()), 0) # Send 21 BTC from 0 to 2 using sendtoaddress call. + # Locked memory should use at least 32 bytes to sign each transaction + self.log.info("test getmemoryinfo") + memory_before = self.nodes[0].getmemoryinfo() self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) - self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) - + mempool_txid = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) + memory_after = self.nodes[0].getmemoryinfo() + assert(memory_before['locked']['used'] + 64 <= memory_after['locked']['used']) + + self.log.info("test gettxout") + # utxo spent in mempool should be visible if you exclude mempool + # but invisible if you include mempool + confirmed_txid, confirmed_index = utxos[0]["txid"], utxos[0]["vout"] + txout = self.nodes[0].gettxout(confirmed_txid, confirmed_index, False) + assert_equal(txout['value'], 50) + txout = self.nodes[0].gettxout(confirmed_txid, confirmed_index, True) + assert txout is None + # new utxo from mempool should be invisible if you exclude mempool + # but visible if you include mempool + txout = self.nodes[0].gettxout(mempool_txid, 0, False) + assert txout is None + txout1 = self.nodes[0].gettxout(mempool_txid, 0, True) + txout2 = self.nodes[0].gettxout(mempool_txid, 1, True) + # note the mempool tx will have randomly assigned indices + # but 10 will go to node2 and the rest will go to node0 + balance = self.nodes[0].getbalance() + assert_equal(set([txout1['value'], txout2['value']]), set([10, balance])) walletinfo = self.nodes[0].getwalletinfo() assert_equal(walletinfo['immature_balance'], 0) @@ -155,7 +178,7 @@ class WalletTest (BitcoinTestFramework): txid2 = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) sync_mempools(self.nodes) - self.nodes.append(start_node(3, self.options.tmpdir, self.extra_args[3])) + self.nodes.append(self.start_node(3, self.options.tmpdir, self.extra_args[3])) connect_nodes_bi(self.nodes, 0, 3) sync_blocks(self.nodes) @@ -198,8 +221,8 @@ class WalletTest (BitcoinTestFramework): assert(found) #do some -walletbroadcast tests - stop_nodes(self.nodes) - self.nodes = start_nodes(3, self.options.tmpdir, [["-walletbroadcast=0"],["-walletbroadcast=0"],["-walletbroadcast=0"]]) + self.stop_nodes() + self.nodes = self.start_nodes(3, self.options.tmpdir, [["-walletbroadcast=0"],["-walletbroadcast=0"],["-walletbroadcast=0"]]) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) @@ -223,8 +246,8 @@ class WalletTest (BitcoinTestFramework): txIdNotBroadcasted = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 2) #restart the nodes with -walletbroadcast=1 - stop_nodes(self.nodes) - self.nodes = start_nodes(3, self.options.tmpdir) + self.stop_nodes() + self.nodes = self.start_nodes(3, self.options.tmpdir) connect_nodes_bi(self.nodes,0,1) connect_nodes_bi(self.nodes,1,2) connect_nodes_bi(self.nodes,0,2) @@ -325,9 +348,9 @@ class WalletTest (BitcoinTestFramework): chainlimit = 6 for m in maintenance: self.log.info("check " + m) - stop_nodes(self.nodes) + self.stop_nodes() # set lower ancestor limit for later - self.nodes = start_nodes(3, self.options.tmpdir, [[m, "-limitancestorcount="+str(chainlimit)]] * 3) + self.nodes = self.start_nodes(3, self.options.tmpdir, [[m, "-limitancestorcount="+str(chainlimit)]] * 3) while m == '-reindex' and [block_count] * 3 != [self.nodes[i].getblockcount() for i in range(3)]: # reindex will leave rpc warm up "early"; Wait for it to finish time.sleep(0.1) @@ -374,8 +397,8 @@ class WalletTest (BitcoinTestFramework): # Try with walletrejectlongchains # Double chain limit but require combining inputs, so we pass SelectCoinsMinConf - stop_node(self.nodes[0],0) - self.nodes[0] = start_node(0, self.options.tmpdir, ["-walletrejectlongchains", "-limitancestorcount="+str(2*chainlimit)]) + self.stop_node(0) + self.nodes[0] = self.start_node(0, self.options.tmpdir, ["-walletrejectlongchains", "-limitancestorcount="+str(2*chainlimit)]) # wait for loadmempool timeout = 10 diff --git a/test/functional/walletbackup.py b/test/functional/walletbackup.py index af1718572f..a4507182a2 100755 --- a/test/functional/walletbackup.py +++ b/test/functional/walletbackup.py @@ -44,14 +44,12 @@ class WalletBackupTest(BitcoinTestFramework): # nodes 1, 2,3 are spenders, let's give them a keypool=100 self.extra_args = [["-keypool=100"], ["-keypool=100"], ["-keypool=100"], []] - # This mirrors how the network was setup in the bash test def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + self.setup_nodes() connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[1], 3) connect_nodes(self.nodes[2], 3) connect_nodes(self.nodes[2], 0) - self.is_network_split=False self.sync_all() def one_send(self, from_node, to_address): @@ -79,18 +77,18 @@ class WalletBackupTest(BitcoinTestFramework): # As above, this mirrors the original bash test. def start_three(self): - self.nodes[0] = start_node(0, self.options.tmpdir) - self.nodes[1] = start_node(1, self.options.tmpdir) - self.nodes[2] = start_node(2, self.options.tmpdir) + self.nodes[0] = self.start_node(0, self.options.tmpdir) + self.nodes[1] = self.start_node(1, self.options.tmpdir) + self.nodes[2] = self.start_node(2, self.options.tmpdir) connect_nodes(self.nodes[0], 3) connect_nodes(self.nodes[1], 3) connect_nodes(self.nodes[2], 3) connect_nodes(self.nodes[2], 0) def stop_three(self): - stop_node(self.nodes[0], 0) - stop_node(self.nodes[1], 1) - stop_node(self.nodes[2], 2) + self.stop_node(0) + self.stop_node(1) + self.stop_node(2) def erase_three(self): os.remove(self.options.tmpdir + "/node0/regtest/wallet.dat") diff --git a/test/functional/zapwallettxes.py b/test/functional/zapwallettxes.py index ce446e44a3..a8600e82f6 100755 --- a/test/functional/zapwallettxes.py +++ b/test/functional/zapwallettxes.py @@ -23,13 +23,9 @@ class ZapWalletTXesTest (BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 - def setup_network(self, split=False): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir) - connect_nodes_bi(self.nodes,0,1) - connect_nodes_bi(self.nodes,1,2) + def setup_network(self): + super().setup_network() connect_nodes_bi(self.nodes,0,2) - self.is_network_split=False - self.sync_all() def run_test (self): self.log.info("Mining blocks...") @@ -62,18 +58,16 @@ class ZapWalletTXesTest (BitcoinTestFramework): assert_equal(tx3['txid'], txid3) #tx3 must be available (unconfirmed) #restart bitcoind - self.nodes[0].stop() - bitcoind_processes[0].wait() - self.nodes[0] = start_node(0,self.options.tmpdir) + self.stop_node(0) + self.nodes[0] = self.start_node(0,self.options.tmpdir) tx3 = self.nodes[0].gettransaction(txid3) assert_equal(tx3['txid'], txid3) #tx must be available (unconfirmed) - self.nodes[0].stop() - bitcoind_processes[0].wait() + self.stop_node(0) #restart bitcoind with zapwallettxes - self.nodes[0] = start_node(0,self.options.tmpdir, ["-zapwallettxes=1"]) + self.nodes[0] = self.start_node(0,self.options.tmpdir, ["-zapwallettxes=1"]) assert_raises(JSONRPCException, self.nodes[0].gettransaction, [txid3]) #there must be a expection because the unconfirmed wallettx0 must be gone by now diff --git a/test/functional/zmq_test.py b/test/functional/zmq_test.py index 9e27b46381..ce39cfefdc 100755 --- a/test/functional/zmq_test.py +++ b/test/functional/zmq_test.py @@ -3,11 +3,12 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ API.""" +import configparser +import os +import struct -from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_framework import BitcoinTestFramework, SkipTest from test_framework.util import * -import zmq -import struct class ZMQTest (BitcoinTestFramework): @@ -18,12 +19,27 @@ class ZMQTest (BitcoinTestFramework): port = 28332 def setup_nodes(self): + # Try to import python3-zmq. Skip this test if the import fails. + try: + import zmq + except ImportError: + raise SkipTest("python3-zmq module not available.") + + # Check that bitcoin has been built with ZMQ enabled + config = configparser.ConfigParser() + if not self.options.configfile: + self.options.configfile = os.path.dirname(__file__) + "/config.ini" + config.read_file(open(self.options.configfile)) + + if not config["components"].getboolean("ENABLE_ZMQ"): + raise SkipTest("bitcoind has not been built with zmq enabled.") + self.zmqContext = zmq.Context() self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashblock") self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashtx") self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % self.port) - return start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[ + self.nodes = self.start_nodes(self.num_nodes, self.options.tmpdir, extra_args=[ ['-zmqpubhashtx=tcp://127.0.0.1:'+str(self.port), '-zmqpubhashblock=tcp://127.0.0.1:'+str(self.port)], [], [], diff --git a/test/util/data/blanktxv1.json b/test/util/data/blanktxv1.json index 51c25a5a98..9fe2de649b 100644 --- a/test/util/data/blanktxv1.json +++ b/test/util/data/blanktxv1.json @@ -2,6 +2,8 @@ "txid": "d21633ba23f70118185227be58a63527675641ad37967e2aa461559f577aec43", "hash": "d21633ba23f70118185227be58a63527675641ad37967e2aa461559f577aec43", "version": 1, + "size": 10, + "vsize": 10, "locktime": 0, "vin": [ ], diff --git a/test/util/data/blanktxv2.json b/test/util/data/blanktxv2.json index 266919f445..e97626e421 100644 --- a/test/util/data/blanktxv2.json +++ b/test/util/data/blanktxv2.json @@ -2,6 +2,8 @@ "txid": "4ebd325a4b394cff8c57e8317ccf5a8d0e2bdf1b8526f8aad6c8e43d8240621a", "hash": "4ebd325a4b394cff8c57e8317ccf5a8d0e2bdf1b8526f8aad6c8e43d8240621a", "version": 2, + "size": 10, + "vsize": 10, "locktime": 0, "vin": [ ], diff --git a/test/util/data/tt-delin1-out.json b/test/util/data/tt-delin1-out.json index 712a2c27f8..f6dfbb51cc 100644 --- a/test/util/data/tt-delin1-out.json +++ b/test/util/data/tt-delin1-out.json @@ -2,6 +2,8 @@ "txid": "81b2035be1da1abe745c6141174a73d151009ec17b3d5ebffa2e177408c50dfd", "hash": "81b2035be1da1abe745c6141174a73d151009ec17b3d5ebffa2e177408c50dfd", "version": 1, + "size": 3040, + "vsize": 3040, "locktime": 0, "vin": [ { diff --git a/test/util/data/tt-delout1-out.json b/test/util/data/tt-delout1-out.json index afc4e95762..6769ed79ff 100644 --- a/test/util/data/tt-delout1-out.json +++ b/test/util/data/tt-delout1-out.json @@ -2,6 +2,8 @@ "txid": "c46ccd75b5050e942b2e86a3648f843f525fe6fc000bf0534ba5973063354493", "hash": "c46ccd75b5050e942b2e86a3648f843f525fe6fc000bf0534ba5973063354493", "version": 1, + "size": 3155, + "vsize": 3155, "locktime": 0, "vin": [ { diff --git a/test/util/data/tt-locktime317000-out.json b/test/util/data/tt-locktime317000-out.json index 2b9075f8ac..82b64df075 100644 --- a/test/util/data/tt-locktime317000-out.json +++ b/test/util/data/tt-locktime317000-out.json @@ -2,6 +2,8 @@ "txid": "aded538f642c17e15f4d3306b8be7e1a4d1ae0c4616d641ab51ea09ba65e5cb5", "hash": "aded538f642c17e15f4d3306b8be7e1a4d1ae0c4616d641ab51ea09ba65e5cb5", "version": 1, + "size": 3189, + "vsize": 3189, "locktime": 317000, "vin": [ { diff --git a/test/util/data/txcreate1.json b/test/util/data/txcreate1.json index f83e036f33..36741044c9 100644 --- a/test/util/data/txcreate1.json +++ b/test/util/data/txcreate1.json @@ -2,6 +2,8 @@ "txid": "fe7d174f42dce0cffa7a527e9bc8368956057619ec817648f6138b98f2533e8f", "hash": "fe7d174f42dce0cffa7a527e9bc8368956057619ec817648f6138b98f2533e8f", "version": 2, + "size": 201, + "vsize": 201, "locktime": 0, "vin": [ { diff --git a/test/util/data/txcreate2.json b/test/util/data/txcreate2.json index fb5e177db7..23fe7ace67 100644 --- a/test/util/data/txcreate2.json +++ b/test/util/data/txcreate2.json @@ -2,6 +2,8 @@ "txid": "0481afb29931341d0d7861d8a2f6f26456fa042abf54a23e96440ed7946e0715", "hash": "0481afb29931341d0d7861d8a2f6f26456fa042abf54a23e96440ed7946e0715", "version": 2, + "size": 19, + "vsize": 19, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatedata1.json b/test/util/data/txcreatedata1.json index 760518d30a..e65a1859eb 100644 --- a/test/util/data/txcreatedata1.json +++ b/test/util/data/txcreatedata1.json @@ -2,6 +2,8 @@ "txid": "07894b4d12fe7853dd911402db1620920d261b9627c447f931417d330c25f06e", "hash": "07894b4d12fe7853dd911402db1620920d261b9627c447f931417d330c25f06e", "version": 1, + "size": 176, + "vsize": 176, "locktime": 0, "vin": [ { diff --git a/test/util/data/txcreatedata2.json b/test/util/data/txcreatedata2.json index 3c6da40f90..8f1544e1c0 100644 --- a/test/util/data/txcreatedata2.json +++ b/test/util/data/txcreatedata2.json @@ -2,6 +2,8 @@ "txid": "c14b007fa3a6c1e7765919c1d14c1cfc2b8642c3a5d3be4b1fa8c4ccfec98bb0", "hash": "c14b007fa3a6c1e7765919c1d14c1cfc2b8642c3a5d3be4b1fa8c4ccfec98bb0", "version": 2, + "size": 176, + "vsize": 176, "locktime": 0, "vin": [ { diff --git a/test/util/data/txcreatedata_seq0.json b/test/util/data/txcreatedata_seq0.json index d272a4c447..e52401f418 100644 --- a/test/util/data/txcreatedata_seq0.json +++ b/test/util/data/txcreatedata_seq0.json @@ -2,6 +2,8 @@ "txid": "8df6ed527472542dd5e137c242a7c5a9f337ac34f7b257ae4af886aeaebb51b0", "hash": "8df6ed527472542dd5e137c242a7c5a9f337ac34f7b257ae4af886aeaebb51b0", "version": 2, + "size": 85, + "vsize": 85, "locktime": 0, "vin": [ { diff --git a/test/util/data/txcreatedata_seq1.json b/test/util/data/txcreatedata_seq1.json index d323255418..093ff4a56b 100644 --- a/test/util/data/txcreatedata_seq1.json +++ b/test/util/data/txcreatedata_seq1.json @@ -2,6 +2,8 @@ "txid": "c4dea671b0d7b48f8ab10bc46650e8329d3c5766931f548f513847a19f5ba75b", "hash": "c4dea671b0d7b48f8ab10bc46650e8329d3c5766931f548f513847a19f5ba75b", "version": 1, + "size": 126, + "vsize": 126, "locktime": 0, "vin": [ { diff --git a/test/util/data/txcreatemultisig1.json b/test/util/data/txcreatemultisig1.json index f6ce43c202..0cc530836a 100644 --- a/test/util/data/txcreatemultisig1.json +++ b/test/util/data/txcreatemultisig1.json @@ -2,6 +2,8 @@ "txid": "0d1d4edfc217d9db3ab6a9298f26a52eae3c52f55a6cb8ccbc14f7c727572894", "hash": "0d1d4edfc217d9db3ab6a9298f26a52eae3c52f55a6cb8ccbc14f7c727572894", "version": 1, + "size": 124, + "vsize": 124, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatemultisig2.json b/test/util/data/txcreatemultisig2.json index e09d22060f..8ad2ffdc65 100644 --- a/test/util/data/txcreatemultisig2.json +++ b/test/util/data/txcreatemultisig2.json @@ -2,6 +2,8 @@ "txid": "0d861f278a3b7bce7cb5a88d71e6e6a903336f95ad5a2c29b295b63835b6eee3", "hash": "0d861f278a3b7bce7cb5a88d71e6e6a903336f95ad5a2c29b295b63835b6eee3", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatemultisig3.json b/test/util/data/txcreatemultisig3.json index 88e32bd310..086bf44b8a 100644 --- a/test/util/data/txcreatemultisig3.json +++ b/test/util/data/txcreatemultisig3.json @@ -2,6 +2,8 @@ "txid": "ccc552220b46a3b5140048b03395987ce4f0fa1ddf8c635bba1fa44e0f8c1d7f", "hash": "ccc552220b46a3b5140048b03395987ce4f0fa1ddf8c635bba1fa44e0f8c1d7f", "version": 1, + "size": 53, + "vsize": 53, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatemultisig4.json b/test/util/data/txcreatemultisig4.json index fc69c7269c..d23ccc045e 100644 --- a/test/util/data/txcreatemultisig4.json +++ b/test/util/data/txcreatemultisig4.json @@ -2,6 +2,8 @@ "txid": "5e8b1cc73234e208d4b7ca9075f136b908c34101be7a048df4ba9ac758b61567", "hash": "5e8b1cc73234e208d4b7ca9075f136b908c34101be7a048df4ba9ac758b61567", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreateoutpubkey1.json b/test/util/data/txcreateoutpubkey1.json index 6019fa2dcd..f10aaecf7a 100644 --- a/test/util/data/txcreateoutpubkey1.json +++ b/test/util/data/txcreateoutpubkey1.json @@ -2,6 +2,8 @@ "txid": "f42b38ac12e3fafc96ba1a9ba70cbfe326744aef75df5fb9db5d6e2855ca415f", "hash": "f42b38ac12e3fafc96ba1a9ba70cbfe326744aef75df5fb9db5d6e2855ca415f", "version": 1, + "size": 54, + "vsize": 54, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreateoutpubkey2.json b/test/util/data/txcreateoutpubkey2.json index 6fc3d57527..5a473b76c3 100644 --- a/test/util/data/txcreateoutpubkey2.json +++ b/test/util/data/txcreateoutpubkey2.json @@ -2,6 +2,8 @@ "txid": "70f2a088cde460e677415fa1fb71895e90c231e6ed38ed203a35b6f848e9cc73", "hash": "70f2a088cde460e677415fa1fb71895e90c231e6ed38ed203a35b6f848e9cc73", "version": 1, + "size": 41, + "vsize": 41, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreateoutpubkey3.json b/test/util/data/txcreateoutpubkey3.json index a1a25fc834..b8389b8f7e 100644 --- a/test/util/data/txcreateoutpubkey3.json +++ b/test/util/data/txcreateoutpubkey3.json @@ -2,6 +2,8 @@ "txid": "bfc7e898ee9f6a9652d7b8cca147e2da134502e2ada0f279ed634fc8cf833f8c", "hash": "bfc7e898ee9f6a9652d7b8cca147e2da134502e2ada0f279ed634fc8cf833f8c", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatescript1.json b/test/util/data/txcreatescript1.json index 8ffecba411..823168e9fb 100644 --- a/test/util/data/txcreatescript1.json +++ b/test/util/data/txcreatescript1.json @@ -2,6 +2,8 @@ "txid": "f0851b68202f736b792649cfc960259c2374badcb644ab20cac726b5f72f61c9", "hash": "f0851b68202f736b792649cfc960259c2374badcb644ab20cac726b5f72f61c9", "version": 1, + "size": 20, + "vsize": 20, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatescript2.json b/test/util/data/txcreatescript2.json index 41eb69f1af..d4c7e10c78 100644 --- a/test/util/data/txcreatescript2.json +++ b/test/util/data/txcreatescript2.json @@ -2,6 +2,8 @@ "txid": "6e07a7cc075e0703f32ee8c4e5373fe654bfbc315148fda364e1be286ff290d0", "hash": "6e07a7cc075e0703f32ee8c4e5373fe654bfbc315148fda364e1be286ff290d0", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatescript3.json b/test/util/data/txcreatescript3.json index 90e7e27f9f..001e69511f 100644 --- a/test/util/data/txcreatescript3.json +++ b/test/util/data/txcreatescript3.json @@ -2,6 +2,8 @@ "txid": "8a234037b088e987c877030efc83374a07441c321bf9dc6dd2f206bc26507df8", "hash": "8a234037b088e987c877030efc83374a07441c321bf9dc6dd2f206bc26507df8", "version": 1, + "size": 53, + "vsize": 53, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatescript4.json b/test/util/data/txcreatescript4.json index 11783751a4..20094bcd44 100644 --- a/test/util/data/txcreatescript4.json +++ b/test/util/data/txcreatescript4.json @@ -2,6 +2,8 @@ "txid": "24225cf5e9391100d6b218134b9f03383ca4c880a1f634ac12990cf28b66adbc", "hash": "24225cf5e9391100d6b218134b9f03383ca4c880a1f634ac12990cf28b66adbc", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatesignv1.json b/test/util/data/txcreatesignv1.json index ff39e71b40..519d3ab066 100644 --- a/test/util/data/txcreatesignv1.json +++ b/test/util/data/txcreatesignv1.json @@ -2,6 +2,8 @@ "txid": "977e7cd286cb72cd470d539ba6cb48400f8f387d97451d45cdb8819437a303af", "hash": "977e7cd286cb72cd470d539ba6cb48400f8f387d97451d45cdb8819437a303af", "version": 1, + "size": 224, + "vsize": 224, "locktime": 0, "vin": [ { |