diff options
175 files changed, 5169 insertions, 1939 deletions
diff --git a/.travis.yml b/.travis.yml index 1f6eb15c27..c8785144ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ os: linux language: cpp +compiler: gcc env: global: - MAKEJOBS=-j3 @@ -30,18 +31,21 @@ matrix: - compiler: ": ARM" env: HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" - compiler: ": Win32" - env: HOST=i686-w64-mingw32 PACKAGES="nsis gcc-mingw-w64-i686 g++-mingw-w64-i686 binutils-mingw-w64-i686 mingw-w64-dev wine bc" RUN_TESTS=true GOAL="deploy" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j2" + env: HOST=i686-w64-mingw32 PPA="ppa:ubuntu-wine/ppa" PACKAGES="nsis gcc-mingw-w64-i686 g++-mingw-w64-i686 binutils-mingw-w64-i686 mingw-w64-dev wine1.7 bc" RUN_TESTS=true GOAL="deploy" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j2" - compiler: ": 32-bit + dash" env: HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" - compiler: ": Win64" - env: HOST=x86_64-w64-mingw32 PACKAGES="nsis gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-dev wine bc" RUN_TESTS=true GOAL="deploy" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j2" + env: HOST=x86_64-w64-mingw32 PPA="ppa:ubuntu-wine/ppa" PACKAGES="nsis gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-dev wine1.7 bc" RUN_TESTS=true GOAL="deploy" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j2" - compiler: ": bitcoind" env: HOST=x86_64-unknown-linux-gnu PACKAGES="bc" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" - compiler: ": No wallet" env: HOST=x86_64-unknown-linux-gnu DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" - compiler: ": Cross-Mac" env: HOST=x86_64-apple-darwin11 PACKAGES="cmake libcap-dev libz-dev libbz2-dev" BITCOIN_CONFIG="--enable-reduce-exports" OSX_SDK=10.9 GOAL="deploy" + exclude: + - compiler: gcc install: + - if [ -n "$PPA" ]; then travis_retry sudo add-apt-repository "$PPA" -y; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi before_script: @@ -1,3 +1,5 @@ +The MIT License (MIT) + Copyright (c) 2009-2015 The Bitcoin Core developers Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/Makefile.am b/Makefile.am index ab68d8fa6d..dfde0d43ec 100644 --- a/Makefile.am +++ b/Makefile.am @@ -204,7 +204,7 @@ endif dist_noinst_SCRIPTS = autogen.sh -EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.sh qa/pull-tester/run-bitcoin-cli qa/rpc-tests $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) +EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.sh qa/rpc-tests $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) diff --git a/configure.ac b/configure.ac index a524bded6f..f0e0a74fe6 100644 --- a/configure.ac +++ b/configure.ac @@ -137,6 +137,12 @@ AC_ARG_ENABLE([glibc-back-compat], [use_glibc_compat=$enableval], [use_glibc_compat=no]) +AC_ARG_ENABLE([zmq], + [AS_HELP_STRING([--disable-zmq], + [Disable ZMQ notifications])], + [use_zmq=$enableval], + [use_zmq=yes]) + AC_ARG_WITH([protoc-bindir],[AS_HELP_STRING([--with-protoc-bindir=BIN_DIR],[specify protoc bin path])], [protoc_bin_path=$withval], []) # Enable debug @@ -664,6 +670,12 @@ if test x$use_pkgconfig = xyes; then if test x$use_qr != xno; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi + if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then + PKG_CHECK_MODULES([EVENT], [libevent],, [AC_MSG_ERROR(libevent not found.)]) + if test x$TARGET_OS != xwindows; then + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)]) + fi + fi ] ) else @@ -673,6 +685,14 @@ else AC_CHECK_HEADER([openssl/ssl.h],, AC_MSG_ERROR(libssl headers missing),) AC_CHECK_LIB([ssl], [main],SSL_LIBS=-lssl, AC_MSG_ERROR(libssl missing)) + if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then + AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) + AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) + if test x$TARGET_OS != xwindows; then + AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) + fi + fi + BITCOIN_QT_CHECK(AC_CHECK_LIB([protobuf] ,[main],[PROTOBUF_LIBS=-lprotobuf], BITCOIN_QT_FAIL(libprotobuf not found))) if test x$use_qr != xno; then BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) @@ -819,6 +839,22 @@ if test x$bitcoin_enable_qt != xno; then fi fi +# conditional search for and use libzmq +AC_MSG_CHECKING([whether to build ZMQ support]) +if test "x$use_zmq" = "xyes"; then + AC_MSG_RESULT([yes]) + PKG_CHECK_MODULES([ZMQ],[libzmq], + [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], + [AC_DEFINE([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) + AC_MSG_WARN([libzmq not found, disabling]) + use_zmq=no]) +else + AC_MSG_RESULT([no, --disable-zmq used]) + AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) +fi + +AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"]) + AC_MSG_CHECKING([whether to build test_bitcoin]) if test x$use_tests = xyes; then AC_MSG_RESULT([yes]) diff --git a/contrib/bitcoin-qt.pro b/contrib/bitcoin-qt.pro index 3a72d10f47..b8133bf789 100644 --- a/contrib/bitcoin-qt.pro +++ b/contrib/bitcoin-qt.pro @@ -11,7 +11,7 @@ FORMS += \ ../src/qt/forms/overviewpage.ui \ ../src/qt/forms/receivecoinsdialog.ui \ ../src/qt/forms/receiverequestdialog.ui \ - ../src/qt/forms/rpcconsole.ui \ + ../src/qt/forms/debugwindow.ui \ ../src/qt/forms/sendcoinsdialog.ui \ ../src/qt/forms/sendcoinsentry.ui \ ../src/qt/forms/signverifymessagedialog.ui \ diff --git a/contrib/bitcoind.bash-completion b/contrib/bitcoind.bash-completion index 3cc959c0a6..1338d2f2b5 100644 --- a/contrib/bitcoind.bash-completion +++ b/contrib/bitcoind.bash-completion @@ -96,7 +96,7 @@ _bitcoind() { esac case "$cur" in - -conf=*|-pid=*|-loadblock=*|-wallet=*|-rpcsslcertificatechainfile=*|-rpcsslprivatekeyfile=*) + -conf=*|-pid=*|-loadblock=*|-wallet=*) cur="${cur#*=}" _filedir return 0 diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 55ebcaab42..d119bbd1d0 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -5,15 +5,11 @@ Upstream-Contact: Satoshi Nakamoto <satoshin@gmx.com> Source: https://github.com/bitcoin/bitcoin Files: * -Copyright: 2009-2012, Bitcoin Core Developers -License: Expat +Copyright: 2009-2015, Bitcoin Core Developers +License: MIT/Expat Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, as well as the numerous contributors to the project. -Files: src/json/* -Copyright: 2007-2009, John W. Wilkinson -License: Expat - Files: debian/* Copyright: 2010-2011, Jonas Smedegaard <dr@jones.dk> 2011, Matt Corallo <matt@bluematt.me> @@ -23,62 +19,68 @@ Files: debian/manpages/* Copyright: Micah Anderson <micah@debian.org> License: GPL-3+ -Files: src/qt/res/icons/clock*.png, src/qt/res/icons/tx*.png, - src/qt/res/src/*.svg -Copyright: Wladimir van der Laan -License: Expat - -Files: src/qt/res/icons/address-book.png, src/qt/res/icons/export.png, - src/qt/res/icons/history.png, src/qt/res/icons/key.png, - src/qt/res/icons/lock_*.png, src/qt/res/icons/overview.png, - src/qt/res/icons/receive.png, src/qt/res/icons/send.png, - src/qt/res/icons/synced.png, src/qt/res/icons/filesave.png -Copyright: David Vignoni (david@icon-king.com) - ICON KING - www.icon-king.com -License: LGPL -Comment: NUVOLA ICON THEME for KDE 3.x - Original icons: kaddressbook, klipper_dock, view-list-text, - key-password, encrypted/decrypted, go-home, go-down, - go-next, dialog-ok - Site: http://www.icon-king.com/projects/nuvola/ +Files: src/qt/res/icons/add.png, + src/qt/res/icons/address-book.png, + src/qt/res/icons/configure.png, + src/qt/res/icons/debugwindow.png, + src/qt/res/icons/edit.png, + src/qt/res/icons/editcopy.png, + src/qt/res/icons/editpaste.png, + src/qt/res/icons/export.png, + src/qt/res/icons/eye.png, + src/qt/res/icons/filesave.png, + src/qt/res/icons/history.png, + src/qt/res/icons/info.png, + src/qt/res/icons/key.png, + src/qt/res/icons/lock_*.png, + src/qt/res/icons/open.png, + src/qt/res/icons/overview.png, + src/qt/res/icons/quit.png, + src/qt/res/icons/receive.png, + src/qt/res/icons/remove.png, + src/qt/res/icons/send.png, + src/qt/res/icons/synced.png, + src/qt/res/icons/transaction*.png, + src/qt/res/icons/tx_output.png, + src/qt/res/icons/warning.png +Copyright: Stephen Hutchings (and more) + http://typicons.com +License: MIT/Expat +Comment: Site: https://github.com/stephenhutchings/typicons.font Files: src/qt/res/icons/connect*.png -Copyright: schollidesign -License: GPL-3+ -Comment: Icon Pack: Human-O2 - Site: http://findicons.com/icon/93743/blocks_gnome_netstatus_0 - -Files: src/qt/res/icons/transaction*.png -Copyright: md2k7 -License: Expat -Comment: Site: https://bitcointalk.org/index.php?topic=15276.0 - -Files: src/qt/res/icons/configure.png, src/qt/res/icons/quit.png, - src/qt/res/icons/editcopy.png, src/qt/res/icons/editpaste.png, - src/qt/res/icons/add.png, src/qt/res/icons/edit.png, - src/qt/res/icons/remove.png -Copyright: http://www.everaldo.com -License: LGPL -Comment: Icon Pack: Crystal SVG + src/qt/res/src/connect-*.svg +Copyright: Marco Falke +License: MIT/Expat +Comment: Inspired by Stephan Hutchings Typicons + +Files: src/qt/res/icons/tx_mined.png + src/qt/res/src/mine.svg +Copyright: Jonas Schnelli +License: MIT/Expat +Comment: -Files: src/qt/res/icons/bitcoin.png, src/qt/res/icons/toolbar.png -Copyright: Bitboy (optimized for 16x16 by Wladimir van der Laan) +Files: src/qt/res/icons/clock*.png + src/qt/res/icons/eye_*.png + src/qt/res/icons/verify.png + src/qt/res/icons/tx_in*.png + src/qt/res/src/clock_*.svg + src/qt/res/src/tx_*.svg + src/qt/res/src/verify.svg +Copyright: Stephan Hutching, Jonas Schnelli +License: MIT/Expat +Comment: Modifications of Stephan Hutchings Typicons + +Files: src/qt/res/icons/about.png + src/qt/res/icons/bitcoin.* + share/pixmaps/bitcoin* + src/qt/res/src/bitcoin.svg +Copyright: Bitboy, Jonas Schnelli License: PUB-DOM Comment: Site: https://bitcointalk.org/?topic=1756.0 -Files: scripts/img/reload.xcf, src/qt/res/movies/*.png -Copyright: Everaldo (Everaldo Coelho) -License: GPL-3+ -Comment: Icon Pack: Kids - Site: http://findicons.com/icon/17102/reload?id=17102 - -Files: src/qt/res/images/splash2.jpg -License: PUB-DOM -Copyright: Crobbo (forum) -Comment: Site: https://bitcointalk.org/index.php?topic=32273.0 - -License: Expat +License: MIT/Expat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including @@ -98,20 +100,6 @@ License: Expat TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -License: ISC - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - . - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL - WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR - BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES - OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, - WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, - ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS - SOFTWARE. - License: GPL-2+ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the @@ -140,22 +128,5 @@ Comment: You should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. -License: LGPL - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - . - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. -Comment: - On Debian systems the GNU Lesser General Public License (LGPL) is - located in '/usr/share/common-licenses/LGPL'. - . - You should have received a copy of the GNU General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. - License: PUB-DOM This work is in the public domain. diff --git a/contrib/debian/examples/bitcoin.conf b/contrib/debian/examples/bitcoin.conf index ade80d60ea..2831c07292 100644 --- a/contrib/debian/examples/bitcoin.conf +++ b/contrib/debian/examples/bitcoin.conf @@ -60,7 +60,7 @@ # JSON-RPC options (for controlling a running Bitcoin/bitcoind process) # -# server=1 tells Bitcoin-QT and bitcoind to accept JSON-RPC commands +# server=1 tells Bitcoin-Qt and bitcoind to accept JSON-RPC commands #server=0 # Bind to given address to listen for JSON-RPC connections. Use [host]:port notation for IPv6. @@ -73,7 +73,7 @@ # How many seconds bitcoin will wait for a complete RPC HTTP request. # after the HTTP connection is established. -#rpctimeout=30 +#rpcclienttimeout=30 # By default, only RPC connections from localhost are allowed. # Specify as many rpcallowip= settings as you like to allow connections from other hosts, @@ -82,7 +82,7 @@ # NOTE: opening up the RPC port to hosts outside your local trusted network is NOT RECOMMENDED, # because the rpcpassword is transmitted over the network unencrypted. -# server=1 tells Bitcoin-QT to accept JSON-RPC commands. +# server=1 tells Bitcoin-Qt to accept JSON-RPC commands. # it is also read by bitcoind to determine if RPC should be enabled #rpcallowip=10.1.1.34/255.255.255.0 #rpcallowip=1.2.3.4/24 @@ -95,15 +95,6 @@ # running on another host using this option: #rpcconnect=127.0.0.1 -# Use Secure Sockets Layer (also known as TLS or HTTPS) to communicate -# with Bitcoin -server or bitcoind -#rpcssl=1 - -# OpenSSL settings used when rpcssl=1 -#rpcsslciphers=TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!AH:!3DES:@STRENGTH -#rpcsslcertificatechainfile=server.cert -#rpcsslprivatekeyfile=server.pem - # Transaction Fee Changes in 0.10.0 # Send transactions as zero-fee transactions if possible (default: 0) diff --git a/contrib/debian/manpages/bitcoin-cli.1 b/contrib/debian/manpages/bitcoin-cli.1 index f953ae9db7..154b458739 100644 --- a/contrib/debian/manpages/bitcoin-cli.1 +++ b/contrib/debian/manpages/bitcoin-cli.1 @@ -36,9 +36,6 @@ Listen for JSON\-RPC connections on <port> (default: 8332 or testnet: 18332). .TP \fB\-rpcconnect=\fR<ip> Send commands to node running on <ip> (default: 127.0.0.1). -.TP -\fB\-rpcssl\fR=\fI1\fR -Use OpenSSL (https) for JSON\-RPC connections (see the Bitcoin Wiki for SSL setup instructions). .SH "SEE ALSO" \fBbitcoind\fP, \fBbitcoin.conf\fP diff --git a/contrib/debian/manpages/bitcoin-qt.1 b/contrib/debian/manpages/bitcoin-qt.1 index a023582bc0..05eadc94cd 100644 --- a/contrib/debian/manpages/bitcoin-qt.1 +++ b/contrib/debian/manpages/bitcoin-qt.1 @@ -178,18 +178,6 @@ Set maximum block size in bytes (default: 250000) .HP \fB\-blockprioritysize=\fR<n> Set maximum size of high\-priority/low\-fee transactions in bytes (default: 27000) .PP -SSL options: (see the Bitcoin Wiki for SSL setup instructions) -.TP -\fB\-rpcssl\fR -Use OpenSSL (https) for JSON\-RPC connections -.TP -\fB\-rpcsslcertificatechainfile=\fR<file.cert> -Server certificate file (default: server.cert) -.TP -\fB\-rpcsslprivatekeyfile=\fR<file.pem> -Server private key (default: server.pem) -.TP -\fB\-rpcsslciphers=\fR<ciphers> Acceptable ciphers (default: TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!AH:!3DES:@STRENGTH) .SS "UI options:" .TP diff --git a/contrib/debian/manpages/bitcoin.conf.5 b/contrib/debian/manpages/bitcoin.conf.5 index 8a0078d5d5..0cf4d991e3 100644 --- a/contrib/debian/manpages/bitcoin.conf.5 +++ b/contrib/debian/manpages/bitcoin.conf.5 @@ -46,16 +46,6 @@ Listen for RPC connections on this TCP port. \fBrpcconnect=\fR\fI'127.0.0.1'\fR You can use *bitcoin* or *bitcoind(1)* to send commands to *bitcoin*/*bitcoind(1)* running on another host using this option. .TP -\fBrpcssl=\fR\fI'1'\fR -Use Secure Sockets Layer (also known as TLS or HTTPS) to communicate with *bitcoin* '\-server' or *bitcoind(1)*. Example of OpenSSL settings used when *rpcssl*='1': -.TP -\fB\-rpcsslciphers=\fR<ciphers> -Acceptable ciphers (default: TLSv1+HIGH:\:!SSLv2:\:!aNULL:\:!eNULL:\:!AH:\:!3DES:\:@STRENGTH) -.TP -\fBrpcsslcertificatechainfile=\fR\fI'server.cert'\fR -.TP -\fBrpcsslprivatekeyfile=\fR\fI'server.pem'\fR -.TP .SH MISCELLANEOUS OPTIONS .TP \fBgen=\fR[\fI'0'\fR|\fI'1'\fR] diff --git a/contrib/debian/manpages/bitcoind.1 b/contrib/debian/manpages/bitcoind.1 index c225b9f3e9..5b0f2921aa 100644 --- a/contrib/debian/manpages/bitcoind.1 +++ b/contrib/debian/manpages/bitcoind.1 @@ -62,20 +62,6 @@ Allow JSON\-RPC connections from specified IP address .TP \fB\-rpcconnect=\fR<ip> Send commands to node running on <ip> -.PP -SSL options: (see the Bitcoin Wiki for SSL setup instructions) -.TP -\fB\-rpcssl\fR=\fI1\fR -Use OpenSSL (https) for JSON\-RPC connections -.TP -\fB\-rpcsslcertificatchainfile=\fR<file.cert> -Server certificate file (default: server.cert) -.TP -\fB\-rpcsslprivatekeyfile=\fR<file.pem> -Server private key (default: server.pem) -.TP -\fB\-rpcsslciphers=\fR<ciphers> -Acceptable ciphers (default: TLSv1+HIGH:\:!SSLv2:\:!aNULL:\:!eNULL:\:!AH:\:!3DES:\:@STRENGTH) .TP \-? This help message diff --git a/contrib/devtools/github-merge.sh b/contrib/devtools/github-merge.sh index ec7a1f4c4b..afb53f0390 100755 --- a/contrib/devtools/github-merge.sh +++ b/contrib/devtools/github-merge.sh @@ -161,7 +161,11 @@ if [[ "d$REPLY" =~ ^d[Ss]$ ]]; then cleanup exit 1 else - git commit -q --gpg-sign --amend --no-edit + if ! git commit -q --gpg-sign --amend --no-edit; then + echo "Error signing, exiting." + cleanup + exit 1 + fi fi else echo "Not signing off on merge, exiting." diff --git a/contrib/verify-commits/trusted-git-root b/contrib/verify-commits/trusted-git-root index eb13f8762e..838b8d1ea8 100644 --- a/contrib/verify-commits/trusted-git-root +++ b/contrib/verify-commits/trusted-git-root @@ -1 +1 @@ -053038e5ba116cb319fb85f3cb3e062cf1b3df15 +165e323d851cc87213c7673c6f278e87a6f2e752 diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py new file mode 100755 index 0000000000..decf29d42a --- /dev/null +++ b/contrib/zmq/zmq_sub.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python2 + +import array +import binascii +import zmq + +port = 28332 + +zmqContext = zmq.Context() +zmqSubSocket = zmqContext.socket(zmq.SUB) +zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashblock") +zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashtx") +zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "rawblock") +zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "rawtx") +zmqSubSocket.connect("tcp://127.0.0.1:%i" % port) + +try: + while True: + msg = zmqSubSocket.recv_multipart() + topic = str(msg[0]) + body = msg[1] + + if topic == "hashblock": + print "- HASH BLOCK -" + print binascii.hexlify(body) + elif topic == "hashtx": + print '- HASH TX -' + print binascii.hexlify(body) + elif topic == "rawblock": + print "- RAW BLOCK HEADER -" + print binascii.hexlify(body[:80]) + elif topic == "rawtx": + print '- RAW TX -' + print binascii.hexlify(body) + +except KeyboardInterrupt: + zmqContext.destroy() diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk new file mode 100644 index 0000000000..2e9be1e98c --- /dev/null +++ b/depends/packages/libevent.mk @@ -0,0 +1,31 @@ +package=libevent +$(package)_version=2.0.22 +$(package)_download_path=https://github.com/libevent/libevent/releases/download/release-2.0.22-stable +$(package)_file_name=$(package)-$($(package)_version)-stable.tar.gz +$(package)_sha256_hash=71c2c49f0adadacfdbe6332a372c38cf9c8b7895bb73dabeaa53cdcc1d4e1fa3 +$(package)_patches=reuseaddr.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/reuseaddr.patch +endef + +define $(package)_set_vars + $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress + $(package)_config_opts_release=--disable-debug-mode + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds +endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 03908aba59..02cc188420 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,4 +1,6 @@ -packages:=boost openssl +packages:=boost openssl libevent +darwin_packages:=zeromq +linux_packages:=zeromq native_packages := native_ccache native_comparisontool qt_native_packages = native_protobuf diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk new file mode 100644 index 0000000000..24e8e5f1c9 --- /dev/null +++ b/depends/packages/zeromq.mk @@ -0,0 +1,26 @@ +package=zeromq +$(package)_version=4.0.4 +$(package)_download_path=http://download.zeromq.org +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=1ef71d46e94f33e27dd5a1661ed626cd39be4d2d6967792a275040e34457d399 + +define $(package)_set_vars + $(package)_config_opts=--without-documentation --disable-shared + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) -C src +endef + +define $(package)_stage_cmds + $(MAKE) -C src DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf bin share +endef diff --git a/depends/patches/libevent/reuseaddr.patch b/depends/patches/libevent/reuseaddr.patch new file mode 100644 index 0000000000..58695c11f5 --- /dev/null +++ b/depends/patches/libevent/reuseaddr.patch @@ -0,0 +1,21 @@ +--- old/evutil.c 2015-08-28 19:26:23.488765923 -0400 ++++ new/evutil.c 2015-08-28 19:27:41.392767019 -0400 +@@ -321,15 +321,16 @@ + int + evutil_make_listen_socket_reuseable(evutil_socket_t sock) + { +-#ifndef WIN32 + int one = 1; ++#ifndef WIN32 + /* REUSEADDR on Unix means, "don't hang on to this address after the + * listener is closed." On Windows, though, it means "don't keep other + * processes from binding to this address while we're using it. */ + return setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*) &one, + (ev_socklen_t)sizeof(one)); + #else +- return 0; ++ return setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*) &one, ++ (ev_socklen_t)sizeof(one)); + #endif + } + diff --git a/doc/assets-attribution.md b/doc/assets-attribution.md index b5a033dd79..2dd930d6a4 100644 --- a/doc/assets-attribution.md +++ b/doc/assets-attribution.md @@ -1,67 +1 @@ -The following is a list of assets used in the bitcoin source and their proper attribution. - -[Typicons/Stephen Hutchings](http://typicons.com) ------------------------ - -### Info -* Icon Pack: Typicons (http://typicons.com) -* Designer: Stephen Hutchings (and more) -* License: MIT -* Site: [https://github.com/stephenhutchings/typicons.font](https://github.com/stephenhutchings/typicons.font) - -### Assets Used - src/qt/res/icons/add.png - src/qt/res/icons/address-book.png, - src/qt/res/icons/configure.png - src/qt/res/icons/debugwindow.png - src/qt/res/icons/edit.png, - src/qt/res/icons/exitcopy.png - src/qt/res/icons/editpaste.png, - src/qt/res/icons/export.png - src/qt/res/icons/eye.png, - src/qt/res/icons/filesave.png - src/qt/res/icons/history.png, - src/qt/res/icons/info.png - src/qt/res/icons/key.png, - src/qt/res/icons/lock_*.png - src/qt/res/icons/open.png, - src/qt/res/icons/overview.png - src/qt/res/icons/quit.png, - src/qt/res/icons/receive.png - src/qt/res/icons/remove.png, - src/qt/res/icons/send.png - src/qt/res/icons/synced.png, - src/qt/res/icons/transaction*.png - src/qt/res/icons/tx_output.png, - src/qt/res/icons/warning.png - -Other ------------------------ - -### Info -* Designer: Jonas Schnelli, Bitboy, Stephen Hutchings, Marco Falke -* Bitcoin Icon: Based on the original bitcoin logo from Bitboy -* Network connection icon: Inspired by flow-merge.svg from Stephen Hutchings -* Some icons are based on Stephan Hutchings Typicons -* License: MIT - -### Assets Used - src/qt/res/icons/about.png - src/qt/res/icons/about_qt.png, - src/qt/res/icons/bitcoin.icns - src/qt/res/icons/bitcoin.ico, - src/qt/res/icons/bitcoin.png - src/qt/res/icons/clock*.png, - src/qt/res/icons/connect*.png - src/qt/res/icons/eye_minus.png, - src/qt/res/icons/eye_plus.png - src/qt/res/icons/verify.png, - src/qt/res/icons/tx_inout.png - src/qt/res/icons/tx_input.png, - src/qt/res/src/bitcoin.svg, - src/qt/res/src/clock*.svg - src/qt/res/src/connect*.svg, - src/qt/res/src/mine.svg - src/qt/res/src/qt.svg - src/qt/res/src/tx*.svg, - src/qt/res/src/verify.svg +The list of assets used in the bitcoin source and their attribution can now be found in [contrib/debian/copyright](../contrib/debian/copyright). diff --git a/doc/bips.md b/doc/bips.md index 90e98ed419..c84bd966f5 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -1,4 +1,4 @@ -BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.10.0**): +BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.12.0**): * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). * [`BIP 13`](https://github.com/bitcoin/bips/blob/master/bip-0013.mediawiki): The address format for P2SH addresses has been implemented since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). @@ -16,3 +16,4 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.10.0**): * [`BIP 61`](https://github.com/bitcoin/bips/blob/master/bip-0061.mediawiki): The 'reject' protocol message (and the protocol version bump to 70002) was added in **v0.9.0** ([PR #3185](https://github.com/bitcoin/bitcoin/pull/3185)). * [`BIP 66`](https://github.com/bitcoin/bips/blob/master/bip-0066.mediawiki): The strict DER rules and associated version 3 blocks have been implemented since **v0.10.0** ([PR #5713](https://github.com/bitcoin/bitcoin/pull/5713)). * [`BIP 70`](https://github.com/bitcoin/bips/blob/master/bip-0070.mediawiki) [`71`](https://github.com/bitcoin/bips/blob/master/bip-0071.mediawiki) [`72`](https://github.com/bitcoin/bips/blob/master/bip-0072.mediawiki): Payment Protocol support has been available in Bitcoin Core GUI since **v0.9.0** ([PR #5216](https://github.com/bitcoin/bitcoin/pull/5216)). +* [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, but only enforced for peer versions `>=70011` as of **v0.12.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579)). diff --git a/doc/build-osx.md b/doc/build-osx.md index dc319dd1c4..201fe9522b 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -32,7 +32,7 @@ Instructions: Homebrew #### Install dependencies using Homebrew - brew install autoconf automake berkeley-db4 libtool boost miniupnpc openssl pkg-config protobuf qt5 + brew install autoconf automake berkeley-db4 libtool boost miniupnpc openssl pkg-config protobuf qt5 libevent NOTE: Building with Qt4 is still supported, however, could result in a broken UI. As such, building with Qt5 is recommended. @@ -70,7 +70,7 @@ Download Qt Creator from http://www.qt.io/download/. Download the "community edi 6. Confirm the "summary page" 7. In the "Projects" tab select "Manage Kits..." 8. Select the default "Desktop" kit and select "Clang (x86 64bit in /usr/bin)" as compiler -9. Select LLDB as debugger (you might need to set the path to your installtion) +9. Select LLDB as debugger (you might need to set the path to your installation) 10. Start debugging with Qt Creator Creating a release build diff --git a/doc/build-unix.md b/doc/build-unix.md index 92aed7725e..e02a5e42f7 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -31,8 +31,9 @@ These dependencies are required: Library | Purpose | Description ------------|------------------|---------------------- - libssl | SSL Support | Secure communications - libboost | Boost | C++ Library + libssl | Crypto | Random Number Generation, Elliptic Curve Cryptography + libboost | Utility | Library for threading, data structures, etc + libevent | Networking | OS independent asynchronous networking Optional dependencies: @@ -57,7 +58,7 @@ Dependency Build Instructions: Ubuntu & Debian ---------------------------------------------- Build requirements: - sudo apt-get install build-essential libtool autotools-dev autoconf pkg-config libssl-dev + sudo apt-get install build-essential libtool autotools-dev autoconf pkg-config libssl-dev libevent-dev For Ubuntu 12.04 and later or Debian 7 and later libboost-all-dev has to be installed: @@ -154,7 +155,8 @@ make install # Configure Bitcoin Core to use our own-built instance of BDB cd $BITCOIN_ROOT -./configure (other args...) LDFLAGS="-L${BDB_PREFIX}/lib/" CPPFLAGS="-I${BDB_PREFIX}/include/" +./autogen.sh +./configure LDFLAGS="-L${BDB_PREFIX}/lib/" CPPFLAGS="-I${BDB_PREFIX}/include/" # (other args...) ``` **Note**: You only need Berkeley DB if the wallet is enabled (see the section *Disable-Wallet mode* below). diff --git a/doc/gitian-building.md b/doc/gitian-building.md index 630b3c04a7..b434ae8a5b 100644 --- a/doc/gitian-building.md +++ b/doc/gitian-building.md @@ -5,7 +5,7 @@ Gitian building Gitian is the deterministic build process that is used to build the Bitcoin Core executables. It provides a way to be reasonably sure that the -executables are really built from source on GitHub. It also makes sure that +executables are really built from the source on GitHub. It also makes sure that the same, tested dependencies are used and statically built into the executable. Multiple developers build the source code by following a specific descriptor @@ -13,8 +13,8 @@ Multiple developers build the source code by following a specific descriptor These results are compared and only if they match, the build is accepted and uploaded to bitcoin.org. -More independent gitian builders are needed, which is why I wrote this -guide. It is preferred to follow these steps yourself instead of using someone else's +More independent gitian builders are needed, which is why this guide exists. +It is preferred you follow these steps yourself instead of using someone else's VM image to avoid 'contaminating' the build. Table of Contents @@ -39,46 +39,46 @@ This guide explains how to set up the environment, and how to start the builds. Debian Linux was chosen as the host distribution because it has a lightweight install (in contrast to Ubuntu) and is readily available. Any kind of virtualization can be used, for example: -- [VirtualBox](https://www.virtualbox.org/), covered by this guide +- [VirtualBox](https://www.virtualbox.org/) (covered by this guide) - [KVM](http://www.linux-kvm.org/page/Main_Page) - [LXC](https://linuxcontainers.org/), see also [Gitian host docker container](https://github.com/gdm85/tenku/tree/master/docker/gitian-bitcoin-host/README.md). -You can also install on actual hardware instead of using virtualization. +You can also install gitian on actual hardware instead of using virtualization. Create a new VirtualBox VM --------------------------- In the VirtualBox GUI click "Create" and choose the following parameters in the wizard: - + -- Type: Linux, Debian (64 bit) +- Type: Linux, Debian (64-bit)  -- Memory Size: at least 1024MB, anything lower will really slow the build down +- Memory Size: at least 1024MB, anything less will really slow down the build. - + -- Hard Drive: Create a virtual hard drive now +- Hard Disk: Create a virtual hard disk now - + -- Hard Drive file type: Use the default, VDI (VirtualBox Disk Image) +- Hard Disk file type: Use the default, VDI (VirtualBox Disk Image) - + -- Storage on Physical hard drive: Dynamically Allocated +- Storage on physical hard disk: Dynamically Allocated  -- Disk size: at least 40GB; as low as 20GB *may* be possible, but better to err on the safe side -- Push the `Create` button +- File location and size: at least 40GB; as low as 20GB *may* be possible, but better to err on the safe side +- Click `Create` -Get the [Debian 7.8 net installer](http://cdimage.debian.org/cdimage/archive/7.8.0/amd64/iso-cd/debian-7.8.0-amd64-netinst.iso) (a more recent minor version should also work, see also [Debian Network installation](https://www.debian.org/CD/netinst/)). +Get the [Debian 8.1 net installer](http://cdimage.debian.org/debian-cd/8.1.0/amd64/iso-cd/debian-8.1.0-amd64-netinst.iso) (a more recent minor version should also work, see also [Debian Network installation](https://www.debian.org/CD/netinst/)). This DVD image can be validated using a SHA256 hashing tool, for example on Unixy OSes by entering the following in a terminal: - echo "e39c36d6adc0fd86c6edb0e03e22919086c883b37ca194d063b8e3e8f6ff6a3a debian-7.8.0-amd64-netinst.iso" | sha256sum -c + echo "5d0a1f804d73aee73eee7efbb38456390558094fd19894a573f1514ca44347e0 debian-8.1.0-amd64-netinst.iso" | sha256sum -c # (must return OK) After creating the VM, we need to configure it. @@ -115,8 +115,9 @@ This section will explain how to install Debian on the newly created VM.  -**Note**: Navigation in the Debian installer: To keep a setting at the default -and proceed, just press `Enter`. To select a different button, press `Tab`. +**Note**: Navigating in the Debian installer: +To keep a setting at the default and proceed, just press `Enter`. +To select a different button, press `Tab`. - Choose locale and keyboard settings (doesn't matter, you can just go with the defaults or select your own information) @@ -126,7 +127,7 @@ and proceed, just press `Enter`. To select a different button, press `Tab`. - The VM will detect network settings using DHCP, this should all proceed automatically - Configure the network: - - System name `debian`. + - Hostname `debian`. - Leave domain name empty.  @@ -136,6 +137,7 @@ and proceed, just press `Enter`. To select a different button, press `Tab`.  - Name the new user `debian` (the full name doesn't matter, you can leave it empty) +- Set the account username as `debian`   @@ -158,13 +160,9 @@ and proceed, just press `Enter`. To select a different button, press `Tab`.  - - Partitioning scheme: All files in one partition - - - - Finish partitioning and write changes to disk -> *Yes* (`Tab`, `Enter` to select the `Yes` button) - +  - The base system will be installed, this will take a minute or so @@ -172,51 +170,79 @@ and proceed, just press `Enter`. To select a different button, press `Tab`.  -- Enter proxy information (unless you are on an intranet, you can leave this empty) +- Enter proxy information (unless you are on an intranet, leave this empty)  - Wait a bit while 'Select and install software' runs - Participate in popularity contest -> *No* -- Choose software to install. We need just the base system. +- Choose software to install. We need just the base system. +- Make sure only 'SSH server' and 'Standard System Utilities' are checked +- Uncheck 'Debian Desktop Environment' and 'Print Server'  -- Make sure only 'SSH server' and 'Standard System Utilities' are checked -- Uncheck 'Debian Desktop Environment' and 'Print Server' +- Install the GRUB boot loader to the master boot record? -> Yes  -- Install the GRUB boot loader to the master boot record? -> Yes +- Device for boot loader installation -> ata-VBOX_HARDDISK - + - Installation Complete -> *Continue* - After installation, the VM will reboot and you will have a working Debian VM. Congratulations! + + + +After Installation +------------------- +The next step in the guide involves logging in as root via SSH. +SSH login for root users is disabled by default, so we'll enable that now. + +Login to the VM using username `root` and the root password you choose earlier. +You'll be presented with a screen similar to this. + + + +Type: + +``` +sed -i 's/^PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config +``` +and press enter. Then, +``` +/etc/init.d/ssh restart +``` +and enter to restart SSH. Logout by typing 'logout' and pressing 'enter'. + Connecting to the VM ---------------------- After the VM has booted you can connect to it using SSH, and files can be copied from and to the VM using a SFTP utility. Connect to `localhost`, port `22222` (or the port configured when installing the VM). -On Windows you can use putty[1] and WinSCP[2]. +On Windows you can use [putty](http://www.chiark.greenend.org.uk/~sgtatham/putty/download.html) and [WinSCP](http://winscp.net/eng/index.php). -For example to connect as `root` from a Linux command prompt use +For example, to connect as `root` from a Linux command prompt use $ ssh root@localhost -p 22222 The authenticity of host '[localhost]:22222 ([127.0.0.1]:22222)' can't be established. - ECDSA key fingerprint is 8e:71:f9:5b:62:46:de:44:01:da:fb:5f:34:b5:f2:18. + RSA key fingerprint is ae:f5:c8:9f:17:c6:c7:1b:c2:1b:12:31:1d:bb:d0:c7. Are you sure you want to continue connecting (yes/no)? yes - Warning: Permanently added '[localhost]:22222' (ECDSA) to the list of known hosts. + Warning: Permanently added '[localhost]:22222' (RSA) to the list of known hosts. root@localhost's password: (enter root password configured during install) - Linux debian 3.2.0-4-amd64 #1 SMP Debian 3.2.54-2 x86_64 + + The programs included with the Debian GNU/Linux system are free software; + the exact distribution terms for each program are described in the + individual files in /usr/share/doc/*/copyright. + + Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent + permitted by applicable law. root@debian:~# Replace `root` with `debian` to log in as user. -[1] http://www.chiark.greenend.org.uk/~sgtatham/putty/download.html -[2] http://winscp.net/eng/index.php - Setting up Debian for gitian building -------------------------------------- @@ -226,13 +252,10 @@ First we need to log in as `root` to set up dependencies and make sure that our user can use the sudo command. Type/paste the following in the terminal: ```bash -apt-get install git ruby sudo apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils +apt-get install git ruby sudo apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils make ubuntu-archive-keyring adduser debian sudo ``` -When you get a colorful screen with a question about the 'LXC directory', just -go with the default (`/var/lib/lxc`). - Then set up LXC and the rest with the following, which is a complex jumble of settings and workarounds: ```bash @@ -255,7 +278,7 @@ reboot ``` At the end the VM is rebooted to make sure that the changes take effect. The steps in this -section need only to be performed once. +section only need to be performed once. Installing gitian ------------------ @@ -300,26 +323,27 @@ cd gitian-builder bin/make-base-vm --lxc --arch amd64 --suite precise ``` -There will be a lot of warnings printed during build of the image. These can be ignored. +There will be a lot of warnings printed during the build of the image. These can be ignored. **Note**: When sudo asks for a password, enter the password for the user *debian* not for *root*. Getting and building the inputs -------------------------------- -Follow the instructions in [doc/release-process.md](release-process.md) in the bitcoin repository -under 'Fetch and build inputs' to install sources which require manual intervention. Also follow -the next step: 'Seed the Gitian sources cache', which will fetch all necessary source files allowing -for gitian to work offline. +Follow the instructions in [doc/release-process.md](release-process.md#fetch-and-build-inputs-first-time-or-when-dependency-versions-change) +in the bitcoin repository under 'Fetch and build inputs' to install sources which require +manual intervention. Also optionally follow the next step: 'Seed the Gitian sources cache +and offline git repositories' which will fetch the remaining files required for building +offline. Building Bitcoin ---------------- To build Bitcoin (for Linux, OSX and Windows) just follow the steps under 'perform -gitian builds' in [doc/release-process.md](release-process.md) in the bitcoin repository. +gitian builds' in [doc/release-process.md](release-process.md#perform-gitian-builds) in the bitcoin repository. -This may take a long time as it also builds the dependencies needed for each descriptor. -These dependencies will be cached after a successful build to avoid rebuilding them where possible. +This may take some time as it will build all the dependencies needed for each descriptor. +These dependencies will be cached after a successful build to avoid rebuilding them when possible. At any time you can check the package installation and build progress with @@ -331,13 +355,13 @@ tail -f var/build.log Output from `gbuild` will look something like Initialized empty Git repository in /home/debian/gitian-builder/inputs/bitcoin/.git/ - remote: Reusing existing pack: 35606, done. - remote: Total 35606 (delta 0), reused 0 (delta 0) - Receiving objects: 100% (35606/35606), 26.52 MiB | 4.28 MiB/s, done. - Resolving deltas: 100% (25724/25724), done. + remote: Counting objects: 57959, done. + remote: Total 57959 (delta 0), reused 0 (delta 0), pack-reused 57958 + Receiving objects: 100% (57959/57959), 53.76 MiB | 484.00 KiB/s, done. + Resolving deltas: 100% (41590/41590), done. From https://github.com/bitcoin/bitcoin ... (new tags, new branch etc) - --- Building for precise x86_64 --- + --- Building for precise amd64 --- Stopping target if it is up Making a new image copy stdin: is not a tty @@ -368,6 +392,57 @@ COMMIT=2014_03_windows_unicode_path ./bin/gbuild --commit bitcoin=${COMMIT} --url bitcoin=${URL} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml ``` +Building fully offline +----------------------- + +For building fully offline including attaching signatures to unsigned builds, the detached-sigs repository +and the bitcoin git repository with the desired tag must both be available locally, and then gbuild must be +told where to find them. It also requires an apt-cacher-ng which is fully-populated but set to offline mode, or +manually disabling gitian-builder's use of apt-get to update the VM build environment. + +To configure apt-cacher-ng as an offline cacher, you will need to first populate its cache with the relevant +files. You must additionally patch target-bin/bootstrap-fixup to set its apt sources to something other than +plain archive.ubuntu.com: us.archive.ubuntu.com works. + +So, if you use LXC: + +```bash +export PATH="$PATH":/path/to/gitian-builder/libexec +export USE_LXC=1 +cd /path/to/gitian-builder +./libexec/make-clean-vm --suite precise --arch amd64 + +LXC_ARCH=amd64 LXC_SUITE=precise on-target -u root apt-get update +LXC_ARCH=amd64 LXC_SUITE=precise on-target -u root \ + -e DEBIAN_FRONTEND=noninteractive apt-get --no-install-recommends -y install \ + $( sed -ne '/^packages:/,/[^-] .*/ {/^- .*/{s/"//g;s/- //;p}}' ../bitcoin/contrib/gitian-descriptors/*|sort|uniq ) +LXC_ARCH=amd64 LXC_SUITE=precise on-target -u root apt-get -q -y purge grub +LXC_ARCH=amd64 LXC_SUITE=precise on-target -u root -e DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade +``` + +And then set offline mode for apt-cacher-ng: + +``` +/etc/apt-cacher-ng/acng.conf +[...] +Offlinemode: 1 +[...] + +service apt-cacher-ng restart +``` + +Then when building, override the remote URLs that gbuild would otherwise pull from the gitian descriptors:: +```bash + +cd /some/root/path/ +git clone https://github.com/bitcoin/bitcoin-detached-sigs.git + +BTCPATH=/some/root/path/bitcoin.git +SIGPATH=/some/root/path/bitcoin-detached-sigs.git + +./bin/gbuild --url bitcoin=${BTCPATH},signature=${SIGPATH} ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml +``` + Signing externally ------------------- diff --git a/doc/gitian-building/create_new_vm.png b/doc/gitian-building/create_new_vm.png Binary files differnew file mode 100644 index 0000000000..dd22428e17 --- /dev/null +++ b/doc/gitian-building/create_new_vm.png diff --git a/doc/gitian-building/create_vm_hard_disk.png b/doc/gitian-building/create_vm_hard_disk.png Binary files differnew file mode 100644 index 0000000000..8e29816fab --- /dev/null +++ b/doc/gitian-building/create_vm_hard_disk.png diff --git a/doc/gitian-building/create_vm_hard_disk_file_type.png b/doc/gitian-building/create_vm_hard_disk_file_type.png Binary files differnew file mode 100644 index 0000000000..a157211cf5 --- /dev/null +++ b/doc/gitian-building/create_vm_hard_disk_file_type.png diff --git a/doc/gitian-building/create_vm_hard_drive.png b/doc/gitian-building/create_vm_hard_drive.png Binary files differdeleted file mode 100644 index a1706e14fd..0000000000 --- a/doc/gitian-building/create_vm_hard_drive.png +++ /dev/null diff --git a/doc/gitian-building/create_vm_hard_drive_file_type.png b/doc/gitian-building/create_vm_hard_drive_file_type.png Binary files differdeleted file mode 100644 index 251b8ee3e2..0000000000 --- a/doc/gitian-building/create_vm_hard_drive_file_type.png +++ /dev/null diff --git a/doc/gitian-building/create_vm_memsize.png b/doc/gitian-building/create_vm_memsize.png Binary files differindex 33717867a5..5abfee5337 100644 --- a/doc/gitian-building/create_vm_memsize.png +++ b/doc/gitian-building/create_vm_memsize.png diff --git a/doc/gitian-building/create_vm_page1.png b/doc/gitian-building/create_vm_page1.png Binary files differdeleted file mode 100644 index edaebc6223..0000000000 --- a/doc/gitian-building/create_vm_page1.png +++ /dev/null diff --git a/doc/gitian-building/create_vm_storage_physical_hard_disk.png b/doc/gitian-building/create_vm_storage_physical_hard_disk.png Binary files differnew file mode 100644 index 0000000000..cee16a6c63 --- /dev/null +++ b/doc/gitian-building/create_vm_storage_physical_hard_disk.png diff --git a/doc/gitian-building/create_vm_storage_physical_hard_drive.png b/doc/gitian-building/create_vm_storage_physical_hard_drive.png Binary files differdeleted file mode 100644 index 987efaa40c..0000000000 --- a/doc/gitian-building/create_vm_storage_physical_hard_drive.png +++ /dev/null diff --git a/doc/gitian-building/debian_install_10_configure_clock.png b/doc/gitian-building/debian_install_10_configure_clock.png Binary files differindex 467c79018e..7cda038ae4 100644 --- a/doc/gitian-building/debian_install_10_configure_clock.png +++ b/doc/gitian-building/debian_install_10_configure_clock.png diff --git a/doc/gitian-building/debian_install_11_partition_disks.png b/doc/gitian-building/debian_install_11_partition_disks.png Binary files differindex 18110734df..2a648c517f 100644 --- a/doc/gitian-building/debian_install_11_partition_disks.png +++ b/doc/gitian-building/debian_install_11_partition_disks.png diff --git a/doc/gitian-building/debian_install_12_choose_disk.png b/doc/gitian-building/debian_install_12_choose_disk.png Binary files differindex a00d4abf17..0f3acc498e 100644 --- a/doc/gitian-building/debian_install_12_choose_disk.png +++ b/doc/gitian-building/debian_install_12_choose_disk.png diff --git a/doc/gitian-building/debian_install_13_partition_scheme.png b/doc/gitian-building/debian_install_13_partition_scheme.png Binary files differdeleted file mode 100644 index 2f80f19b63..0000000000 --- a/doc/gitian-building/debian_install_13_partition_scheme.png +++ /dev/null diff --git a/doc/gitian-building/debian_install_14_finish.png b/doc/gitian-building/debian_install_14_finish.png Binary files differindex 411d457e95..c8ef0b37ad 100644 --- a/doc/gitian-building/debian_install_14_finish.png +++ b/doc/gitian-building/debian_install_14_finish.png diff --git a/doc/gitian-building/debian_install_15_write_changes.png b/doc/gitian-building/debian_install_15_write_changes.png Binary files differindex f26093982c..d8de00dec6 100644 --- a/doc/gitian-building/debian_install_15_write_changes.png +++ b/doc/gitian-building/debian_install_15_write_changes.png diff --git a/doc/gitian-building/debian_install_16_choose_a_mirror.png b/doc/gitian-building/debian_install_16_choose_a_mirror.png Binary files differindex d2c2e9523b..0bd985b38c 100644 --- a/doc/gitian-building/debian_install_16_choose_a_mirror.png +++ b/doc/gitian-building/debian_install_16_choose_a_mirror.png diff --git a/doc/gitian-building/debian_install_17_choose_a_mirror2.png b/doc/gitian-building/debian_install_17_choose_a_mirror2.png Binary files differdeleted file mode 100644 index cef2db0781..0000000000 --- a/doc/gitian-building/debian_install_17_choose_a_mirror2.png +++ /dev/null diff --git a/doc/gitian-building/debian_install_18_proxy_settings.png b/doc/gitian-building/debian_install_18_proxy_settings.png Binary files differindex 24ba25c109..2c19919f64 100644 --- a/doc/gitian-building/debian_install_18_proxy_settings.png +++ b/doc/gitian-building/debian_install_18_proxy_settings.png diff --git a/doc/gitian-building/debian_install_19_software_selection.png b/doc/gitian-building/debian_install_19_software_selection.png Binary files differindex d462757aff..5430456b14 100644 --- a/doc/gitian-building/debian_install_19_software_selection.png +++ b/doc/gitian-building/debian_install_19_software_selection.png diff --git a/doc/gitian-building/debian_install_1_boot_menu.png b/doc/gitian-building/debian_install_1_boot_menu.png Binary files differindex 27fd849b4f..216502e1c6 100644 --- a/doc/gitian-building/debian_install_1_boot_menu.png +++ b/doc/gitian-building/debian_install_1_boot_menu.png diff --git a/doc/gitian-building/debian_install_20_install_grub.png b/doc/gitian-building/debian_install_20_install_grub.png Binary files differindex de4f9be0c9..d853c15871 100644 --- a/doc/gitian-building/debian_install_20_install_grub.png +++ b/doc/gitian-building/debian_install_20_install_grub.png diff --git a/doc/gitian-building/debian_install_21_install_grub_bootloader.png b/doc/gitian-building/debian_install_21_install_grub_bootloader.png Binary files differnew file mode 100644 index 0000000000..493ab806a6 --- /dev/null +++ b/doc/gitian-building/debian_install_21_install_grub_bootloader.png diff --git a/doc/gitian-building/debian_install_21_finish_installation.png b/doc/gitian-building/debian_install_22_finish_installation.png Binary files differindex b967c3550d..7c4445585b 100644 --- a/doc/gitian-building/debian_install_21_finish_installation.png +++ b/doc/gitian-building/debian_install_22_finish_installation.png diff --git a/doc/gitian-building/debian_install_2_select_a_language.png b/doc/gitian-building/debian_install_2_select_a_language.png Binary files differindex 1c9e0bcfc1..0228ae2c01 100644 --- a/doc/gitian-building/debian_install_2_select_a_language.png +++ b/doc/gitian-building/debian_install_2_select_a_language.png diff --git a/doc/gitian-building/debian_install_3_select_location.png b/doc/gitian-building/debian_install_3_select_location.png Binary files differindex 005c395656..7b18fba975 100644 --- a/doc/gitian-building/debian_install_3_select_location.png +++ b/doc/gitian-building/debian_install_3_select_location.png diff --git a/doc/gitian-building/debian_install_4_configure_keyboard.png b/doc/gitian-building/debian_install_4_configure_keyboard.png Binary files differindex 580c8af7c5..8e46117de4 100644 --- a/doc/gitian-building/debian_install_4_configure_keyboard.png +++ b/doc/gitian-building/debian_install_4_configure_keyboard.png diff --git a/doc/gitian-building/debian_install_5_configure_the_network.png b/doc/gitian-building/debian_install_5_configure_the_network.png Binary files differindex a7fdffc66b..8e3720f243 100644 --- a/doc/gitian-building/debian_install_5_configure_the_network.png +++ b/doc/gitian-building/debian_install_5_configure_the_network.png diff --git a/doc/gitian-building/debian_install_6a_set_up_root_password.png b/doc/gitian-building/debian_install_6a_set_up_root_password.png Binary files differindex 31bd210f38..dcade11967 100644 --- a/doc/gitian-building/debian_install_6a_set_up_root_password.png +++ b/doc/gitian-building/debian_install_6a_set_up_root_password.png diff --git a/doc/gitian-building/debian_install_7_set_up_user_fullname.png b/doc/gitian-building/debian_install_7_set_up_user_fullname.png Binary files differindex bffc6ccd7a..6763c6e08a 100644 --- a/doc/gitian-building/debian_install_7_set_up_user_fullname.png +++ b/doc/gitian-building/debian_install_7_set_up_user_fullname.png diff --git a/doc/gitian-building/debian_install_8_set_up_username.png b/doc/gitian-building/debian_install_8_set_up_username.png Binary files differindex 9e2750ad4e..bb04de96d2 100644 --- a/doc/gitian-building/debian_install_8_set_up_username.png +++ b/doc/gitian-building/debian_install_8_set_up_username.png diff --git a/doc/gitian-building/debian_install_9_user_password.png b/doc/gitian-building/debian_install_9_user_password.png Binary files differindex a26d30cba5..981f1181d7 100644 --- a/doc/gitian-building/debian_install_9_user_password.png +++ b/doc/gitian-building/debian_install_9_user_password.png diff --git a/doc/gitian-building/debian_root_login.png b/doc/gitian-building/debian_root_login.png Binary files differnew file mode 100644 index 0000000000..14cdd5ba5b --- /dev/null +++ b/doc/gitian-building/debian_root_login.png diff --git a/doc/gitian-building/network_settings.png b/doc/gitian-building/network_settings.png Binary files differindex 1d9b6428a7..9e714fd154 100644 --- a/doc/gitian-building/network_settings.png +++ b/doc/gitian-building/network_settings.png diff --git a/doc/gitian-building/port_forwarding_rules.png b/doc/gitian-building/port_forwarding_rules.png Binary files differindex e45c9efffc..9e1fa2af20 100644 --- a/doc/gitian-building/port_forwarding_rules.png +++ b/doc/gitian-building/port_forwarding_rules.png diff --git a/doc/gitian-building/select_startup_disk.png b/doc/gitian-building/select_startup_disk.png Binary files differindex 729b368fd1..5acdc3fe10 100644 --- a/doc/gitian-building/select_startup_disk.png +++ b/doc/gitian-building/select_startup_disk.png diff --git a/doc/release-notes.md b/doc/release-notes.md index 9caac4f7ae..bef9af60ef 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -4,6 +4,59 @@ release-notes at release time) Notable changes =============== +SSL support for RPC dropped +---------------------------- + +SSL support for RPC, previously enabled by the option `rpcssl` has been dropped +from both the client and the server. This was done in preparation for removing +the dependency on OpenSSL for the daemon completely. + +Trying to use `rpcssl` will result in an error: + + Error: SSL mode for RPC (-rpcssl) is no longer supported. + +If you are one of the few people that relies on this feature, a flexible +migration path is to use `stunnel`. This is an utility that can tunnel +arbitrary TCP connections inside SSL. On e.g. Ubuntu it can be installed with: + + sudo apt-get install stunnel4 + +Then, to tunnel a SSL connection on 28332 to a RPC server bound on localhost on port 18332 do: + + stunnel -d 28332 -r 127.0.0.1:18332 -p stunnel.pem -P '' + +It can also be set up system-wide in inetd style. + +Another way to re-attain SSL would be to setup a httpd reverse proxy. This solution +would allow the use of different authentication, loadbalancing, on-the-fly compression and +caching. A sample config for apache2 could look like: + + Listen 443 + + NameVirtualHost *:443 + <VirtualHost *:443> + + SSLEngine On + SSLCertificateFile /etc/apache2/ssl/server.crt + SSLCertificateKeyFile /etc/apache2/ssl/server.key + + <Location /bitcoinrpc> + ProxyPass http://127.0.0.1:8332/ + ProxyPassReverse http://127.0.0.1:8332/ + # optional enable digest auth + # AuthType Digest + # ... + + # optional bypass bitcoind rpc basic auth + # RequestHeader set Authorization "Basic <hash>" + # get the <hash> from the shell with: base64 <<< bitcoinrpc:<password> + </Location> + + # Or, balance the load: + # ProxyPass / balancer://balancer_cluster_name + + </VirtualHost> + Random-cookie RPC authentication --------------------------------- @@ -35,6 +88,32 @@ specified. It used to be the case that `-X -noX` ends up, unintuitively, with X set, as `-X` had precedence over `-noX`. This is no longer the case. Like for other software, the last specified value for an option will hold. +`NODE_BLOOM` service bit +------------------------ + +Support for the `NODE_BLOOM` service bit, as described in [BIP +111](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki), has been +added to the P2P protocol code. + +BIP 111 defines a service bit to allow peers to advertise that they support +bloom filters (such as used by SPV clients) explicitly. It also bumps the protocol +version to allow peers to identify old nodes which allow bloom filtering of the +connection despite lacking the new service bit. + +In this version, it is only enforced for peers that send protocol versions +`>=70011`. For the next major version it is planned that this restriction will be +removed. It is recommended to update SPV clients to check for the `NODE_BLOOM` +service bit for nodes that report versions newer than 70011. + +Merkle branches removed from wallet +----------------------------------- + +Previously, every wallet transaction stored a Merkle branch to prove its +presence in blocks. This wasn't being used for more than an expensive +sanity check. Since 0.12, these are no longer stored. When loading a +0.12 wallet into an older version, it will automatically rescan to avoid +failed checks. + 0.12.0 Change log ================= diff --git a/doc/release-notes/release-notes-0.10.1.md b/doc/release-notes/release-notes-0.10.1.md index 5e939600a0..8f59f1f68c 100644 --- a/doc/release-notes/release-notes-0.10.1.md +++ b/doc/release-notes/release-notes-0.10.1.md @@ -101,7 +101,7 @@ Tests: Miscellaneous: - `c9e022b` Initialization: set Boost path locale in main thread - `23126a0` Sanitize command strings before logging them. -- `323de27` Initialization: setup environment before starting QT tests +- `323de27` Initialization: setup environment before starting Qt tests - `7494e09` Initialization: setup environment before starting tests - `df45564` Initialization: set fallback locale as environment variable diff --git a/doc/release-process.md b/doc/release-process.md index 5ecb9334f5..1bfdb8fabd 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -6,39 +6,54 @@ Release Process * * * -###update (commit) version in sources +###first time only or for new builders, check out the source in the following directory hierarchy + cd /path/to/your/toplevel/build + git clone https://github.com/bitcoin/gitian.sigs.git + git clone https://github.com/devrandom/gitian-builder.git + git clone https://github.com/bitcoin/bitcoin.git + +###for bitcoin maintainers/release engineers, update (commit) version in sources + + pushd ./bitcoin contrib/verifysfbinaries/verify.sh doc/README* share/setup.nsi src/clientversion.h (change CLIENT_VERSION_IS_RELEASE to true) -###tag version in git +###for bitcoin maintainers/release engineers, tag version in git git tag -s v(new version, e.g. 0.8.0) -###write release notes. git shortlog helps a lot, for example: +###for bitcoin maintainers/release engineers, write release notes. git shortlog helps a lot, for example: git shortlog --no-merges v(current version, e.g. 0.7.2)..v(new version, e.g. 0.8.0) + popd * * * -###update gitian - - In order to take advantage of the new caching features in gitian, be sure to update to a recent version (`e9741525c` or later is recommended) +###update gitian, gitian.sigs, checkout bitcoin version, and perform gitian builds -###perform gitian builds - - From a directory containing the bitcoin source, gitian-builder and gitian.sigs + To ensure your gitian descriptors are accurate for direct reference for gbuild, below, run the following from a directory containing the bitcoin source: + pushd ./bitcoin export SIGNER=(your gitian key, ie bluematt, sipa, etc) export VERSION=(new version, e.g. 0.8.0) - pushd ./bitcoin git checkout v${VERSION} popd + + Ensure your gitian.sigs are up-to-date if you wish to gverify your builds against other gitian signatures: + + pushd ./gitian.sigs + git pull + popd + + Ensure your gitian-builder sources are up-to-date to take advantage of the new caching features of gitian (`e9741525c` or later is recommended) + pushd ./gitian-builder + git pull -###fetch and build inputs: (first time, or when dependency versions change) +###fetch and create inputs: (first time, or when dependency versions change) mkdir -p inputs wget -P inputs https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch @@ -52,28 +67,44 @@ Release Process tar -C /Volumes/Xcode/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.9.sdk.tar.gz MacOSX10.9.sdk -###Optional: Seed the Gitian sources cache +###Optional: Seed the Gitian sources cache and offline git repositories - By default, gitian will fetch source files as needed. For offline builds, they can be fetched ahead of time: +By default, gitian will fetch source files as needed. To cache them ahead of time: make -C ../bitcoin/depends download SOURCES_PATH=`pwd`/cache/common - Only missing files will be fetched, so this is safe to re-run for each build. +Only missing files will be fetched, so this is safe to re-run for each build. + +Clone the detached-sigs repository: + + popd + git clone https://github.com/bitcoin/bitcoin-detached-sigs.git + pushd ./bitcoin-builder + +NOTE: Offline builds must use the --url flag to ensure gitian fetches only from local URLs. +For example: ./bin/bguild --url bitcoin=/path/to/bitcoin,signature=/path/to/sigs {rest of arguments} +The following gbuild invocations DO NOT DO THIS by default. -###Build Bitcoin Core for Linux, Windows, and OS X: +###Build (and optionally verify) Bitcoin Core for Linux, Windows, and OS X: ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-linux --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-linux ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../ + ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-win-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win.yml + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-win.yml mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ../ + ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../ popd + Build output expected: 1. source tarball (bitcoin-${VERSION}.tar.gz) @@ -98,19 +129,21 @@ Commit your signature to gitian.sigs: Once the Windows/OSX builds each have 3 matching signatures, they will be signed with their respective release keys. Detached signatures will then be committed to the bitcoin-detached-sigs repository, which can be combined with the unsigned apps to create signed binaries. - Create the signed OSX binary: + Create (and optionally verify) the signed OSX binary: pushd ./gitian-builder ./bin/gbuild -i --commit signature=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml mv build/out/bitcoin-osx-signed.dmg ../bitcoin-${VERSION}-osx.dmg popd - Create the signed Windows binaries: + Create (and optionally verify) the signed Windows binaries: pushd ./gitian-builder ./bin/gbuild -i --commit signature=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-win-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml + ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-signed ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml mv build/out/bitcoin-*win64-setup.exe ../bitcoin-${VERSION}-win64-setup.exe mv build/out/bitcoin-*win32-setup.exe ../bitcoin-${VERSION}-win32-setup.exe popd diff --git a/doc/tor.md b/doc/tor.md index 560f71fa27..f8b94d19d1 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -70,9 +70,14 @@ In a typical situation, where you're only reachable via Tor, this should suffice ./bitcoind -proxy=127.0.0.1:9050 -externalip=57qr3yd1nyntf5k.onion -listen -(obviously, replace the Onion address with your own). If you don't care too much -about hiding your node, and want to be reachable on IPv4 as well, additionally -specify: +(obviously, replace the Onion address with your own). It should be noted that you still +listen on all devices and another node could establish a clearnet connection, when knowing +your address. To mitigate this, additionally bind the address of your Tor proxy: + + ./bitcoind ... -bind=127.0.0.1 + +If you don't care too much about hiding your node, and want to be reachable on IPv4 +as well, use `discover` instead: ./bitcoind ... -discover diff --git a/doc/zmq.md b/doc/zmq.md new file mode 100644 index 0000000000..fd04f6d9f0 --- /dev/null +++ b/doc/zmq.md @@ -0,0 +1,98 @@ +# Block and Transaction Broadcasting With ZeroMQ + +[ZeroMQ](http://zeromq.org/) is a lightweight wrapper around TCP +connections, inter-process communications, and shared-memory, +providing various message-oriented semantics such as publish/subcribe, +request/reply, and push/pull. + +The Bitcoin Core daemon can be configured to act as a trusted "border +router", implementing the bitcoin wire protocol and relay, making +consensus decisions, maintaining the local blockchain database, +broadcasting locally generated transactions into the network, and +providing a queryable RPC interface to interact on a polled basis for +requesting blockchain related data. However, there exists only a +limited service to notify external software of events like the arrival +of new blocks or transactions. + +The ZeroMQ facility implements a notification interface through a +set of specific notifiers. Currently there are notifiers that publish +blocks and transactions. This read-only facility requires only the +connection of a corresponding ZeroMQ subscriber port in receiving +software; it is not authenticated nor is there any two-way protocol +involvement. Therefore, subscribers should validate the received data +since it may be out of date, incomplete or even invalid. + +ZeroMQ sockets are self-connecting and self-healing; that is, connects +made between two endpoints will be automatically restored after an +outage, and either end may be freely started or stopped in any order. + +Because ZeroMQ is message oriented, subscribers receive transactions +and blocks all-at-once and do not need to implement any sort of +buffering or reassembly. + +## Prerequisites + +The ZeroMQ feature in Bitcoin Core uses only a very small part of the +ZeroMQ C API, and is thus compatible with any version of ZeroMQ +from 2.1 onward, including all versions in the 3.x and 4.x release +series. Typically, it is packaged by distributions as something like +*libzmq-dev*. + +The C++ wrapper for ZeroMQ is *not* needed. + +## Enabling + +By default, the ZeroMQ port functionality is enabled. Two steps are +required to enable--compiling in the ZeroMQ code, and configuring +runtime operation on the command-line or configuration file. + + $ ./configure --enable-zmq (other options) + +This will produce a binary that is capable of providing the ZeroMQ +facility, but will not do so until also configured properly. + +## Usage + +Currently, the following notifications are supported: + + -zmqpubhashtx=address + -zmqpubhashblock=address + -zmqpubrawblock=address + -zmqpubrawtx=address + +The socket type is PUB and the address must be a valid ZeroMQ +socket address. The same address can be used in more than one notification. + +For instance: + + $ bitcoind -zmqpubhashtx=tcp://127.0.0.1:28332 -zmqpubrawtx=ipc:///tmp/bitcoind.tx.raw + +Each PUB notification has a topic and body, where the header +corresponds to the notification type. For instance, for the notification +`-zmqpubhashtx` the topic is `hashtx` (no null terminator) and the body is the +hexadecimal transaction hash (32 bytes). + +These options can also be provided in bitcoin.conf. + +ZeroMQ endpoint specifiers for TCP (and others) are documented in the +[ZeroMQ API](http://api.zeromq.org). + +Client side, then, the ZeroMQ subscriber socket must have the +ZMQ_SUBSCRIBE option set to one or either of these prefixes (for instance, just `hash`); without +doing so will result in no messages arriving. Please see `contrib/zmq/zmq_sub.py` +for a working example. + +## Remarks + +From the perspective of bitcoind, the ZeroMQ socket is write-only; PUB +sockets don't even have a read function. Thus, there is no state +introduced into bitcoind directly. Furthermore, no information is +broadcast that wasn't already received from the public P2P network. + +No authentication or authorization is done on connecting clients; it +is assumed that the ZeroMQ port is exposed only to trusted entities, +using other means such as firewalling. + +Note that when the block chain tip changes, a reorganisation may occur and just +the tip will be notified. It is up to the subscriber to retrieve the chain +from the last known block to the new tip. diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index 555d2c224b..2e8a7c69ce 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -5,8 +5,13 @@ CURDIR=$(cd $(dirname "$0"); pwd) # Get BUILDDIR and REAL_BITCOIND . "${CURDIR}/tests-config.sh" -export BITCOINCLI=${BUILDDIR}/qa/pull-tester/run-bitcoin-cli export BITCOIND=${REAL_BITCOIND} +export BITCOINCLI=${REAL_BITCOINCLI} + +if [ "x${EXEEXT}" = "x.exe" ]; then + echo "Win tests currently disabled" + exit 0 +fi #Run the tests @@ -52,8 +57,13 @@ testScriptsExt=( 'invalidblockrequest.py' # 'forknotify.py' 'p2p-acceptblock.py' + 'mempool_packages.py' ); +#if [ "x$ENABLE_ZMQ" = "x1" ]; then +# testScripts+=('zmq_test.py') +#fi + extArg="-extended" passOn=${@#$extArg} diff --git a/qa/pull-tester/run-bitcoin-cli b/qa/pull-tester/run-bitcoin-cli deleted file mode 100755 index 93c25bb9fc..0000000000 --- a/qa/pull-tester/run-bitcoin-cli +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# This is a thin wrapper around bitcoin-cli that strips the Windows-style EOLs -# from the output if present. It is necessary when using bitcoin-cli.exe on -# Linux since shells will interpret the line-endings as part of the result. - -CURDIR=$(cd $(dirname "$0"); pwd) -# Get BUILDDIR and REAL_BITCOIND - -# Grab the value of $REAL_BITCOINCLI which may be bitcoin-cli.exe. -. "${CURDIR}/tests-config.sh" - -"${REAL_BITCOINCLI}" "$@" | sed 's/\r//' diff --git a/qa/pull-tester/tests-config.sh.in b/qa/pull-tester/tests-config.sh.in index 10f4d33e47..e881a95110 100755 --- a/qa/pull-tester/tests-config.sh.in +++ b/qa/pull-tester/tests-config.sh.in @@ -10,6 +10,7 @@ EXEEXT="@EXEEXT@" @ENABLE_WALLET_TRUE@ENABLE_WALLET=1 @BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=1 @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=1 +@ENABLE_ZMQ_TRUE@ENABLE_ZMQ=1 REAL_BITCOIND="$BUILDDIR/src/bitcoind${EXEEXT}" REAL_BITCOINCLI="$BUILDDIR/src/bitcoin-cli${EXEEXT}" diff --git a/qa/rpc-tests/httpbasics.py b/qa/rpc-tests/httpbasics.py index 8ccb821286..b66533543d 100755 --- a/qa/rpc-tests/httpbasics.py +++ b/qa/rpc-tests/httpbasics.py @@ -22,7 +22,7 @@ except ImportError: class HTTPBasicsTest (BitcoinTestFramework): def setup_nodes(self): - return start_nodes(4, self.options.tmpdir, extra_args=[['-rpckeepalive=1'], ['-rpckeepalive=0'], [], []]) + return start_nodes(4, self.options.tmpdir) def run_test(self): @@ -84,9 +84,8 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read(); assert_equal('"error":null' in out1, True) - assert_equal(conn.sock!=None, False) #connection must be closed because keep-alive was set to false - #node2 (third node) is running with standard keep-alive parameters which means keep-alive is off + #node2 (third node) is running with standard keep-alive parameters which means keep-alive is on urlNode2 = urlparse.urlparse(self.nodes[2].url) authpair = urlNode2.username + ':' + urlNode2.password headers = {"Authorization": "Basic " + base64.b64encode(authpair)} diff --git a/qa/rpc-tests/mempool_packages.py b/qa/rpc-tests/mempool_packages.py new file mode 100755 index 0000000000..6bc6e43f0b --- /dev/null +++ b/qa/rpc-tests/mempool_packages.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# Test descendant package tracking code + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +def satoshi_round(amount): + return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) + +class MempoolPackagesTest(BitcoinTestFramework): + + def setup_network(self): + self.nodes = [] + self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", "-relaypriority=0", "-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-maxorphantx=1000", "-relaypriority=0", "-limitancestorcount=5", "-debug"])) + connect_nodes(self.nodes[0], 1) + self.is_network_split = False + self.sync_all() + + # Build a transaction that spends parent_txid:vout + # Return amount sent + def chain_transaction(self, node, parent_txid, vout, value, fee, num_outputs): + send_value = satoshi_round((value - fee)/num_outputs) + inputs = [ {'txid' : parent_txid, 'vout' : vout} ] + outputs = {} + for i in xrange(num_outputs): + outputs[node.getnewaddress()] = send_value + rawtx = node.createrawtransaction(inputs, outputs) + signedtx = node.signrawtransaction(rawtx) + txid = node.sendrawtransaction(signedtx['hex']) + fulltx = node.getrawtransaction(txid, 1) + assert(len(fulltx['vout']) == num_outputs) # make sure we didn't generate a change output + return (txid, send_value) + + def run_test(self): + ''' Mine some blocks and have them mature. ''' + self.nodes[0].generate(101) + utxo = self.nodes[0].listunspent(10) + txid = utxo[0]['txid'] + vout = utxo[0]['vout'] + value = utxo[0]['amount'] + + fee = Decimal("0.0001") + # 100 transactions off a confirmed tx should be fine + chain = [] + for i in xrange(100): + (txid, sent_value) = self.chain_transaction(self.nodes[0], txid, 0, value, fee, 1) + value = sent_value + chain.append(txid) + + # Check mempool has 100 transactions in it, and descendant + # count and fees should look correct + mempool = self.nodes[0].getrawmempool(True) + assert_equal(len(mempool), 100) + descendant_count = 1 + descendant_fees = 0 + descendant_size = 0 + SATOSHIS = 100000000 + + for x in reversed(chain): + assert_equal(mempool[x]['descendantcount'], descendant_count) + descendant_fees += mempool[x]['fee'] + assert_equal(mempool[x]['descendantfees'], SATOSHIS*descendant_fees) + descendant_size += mempool[x]['size'] + assert_equal(mempool[x]['descendantsize'], descendant_size) + descendant_count += 1 + + # Adding one more transaction on to the chain should fail. + try: + self.chain_transaction(self.nodes[0], txid, vout, value, fee, 1) + except JSONRPCException as e: + print "too-long-ancestor-chain successfully rejected" + + # TODO: check that node1's mempool is as expected + + # TODO: test ancestor size limits + + # Now test descendant chain limits + txid = utxo[1]['txid'] + value = utxo[1]['amount'] + vout = utxo[1]['vout'] + + transaction_package = [] + # First create one parent tx with 10 children + (txid, sent_value) = self.chain_transaction(self.nodes[0], txid, vout, value, fee, 10) + parent_transaction = txid + for i in xrange(10): + transaction_package.append({'txid': txid, 'vout': i, 'amount': sent_value}) + + for i in xrange(1000): + utxo = transaction_package.pop(0) + try: + (txid, sent_value) = self.chain_transaction(self.nodes[0], utxo['txid'], utxo['vout'], utxo['amount'], fee, 10) + for j in xrange(10): + transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value}) + if i == 998: + mempool = self.nodes[0].getrawmempool(True) + assert_equal(mempool[parent_transaction]['descendantcount'], 1000) + except JSONRPCException as e: + print e.error['message'] + assert_equal(i, 999) + print "tx that would create too large descendant package successfully rejected" + + # TODO: check that node1's mempool is as expected + + # TODO: test descendant size limits + + # Test reorg handling + # First, the basics: + self.nodes[0].generate(1) + sync_blocks(self.nodes) + self.nodes[1].invalidateblock(self.nodes[0].getbestblockhash()) + self.nodes[1].reconsiderblock(self.nodes[0].getbestblockhash()) + + # Now test the case where node1 has a transaction T in its mempool that + # depends on transactions A and B which are in a mined block, and the + # block containing A and B is disconnected, AND B is not accepted back + # into node1's mempool because its ancestor count is too high. + + # Create 8 transactions, like so: + # Tx0 -> Tx1 (vout0) + # \--> Tx2 (vout1) -> Tx3 -> Tx4 -> Tx5 -> Tx6 -> Tx7 + # + # Mine them in the next block, then generate a new tx8 that spends + # Tx1 and Tx7, and add to node1's mempool, then disconnect the + # last block. + + # Create tx0 with 2 outputs + utxo = self.nodes[0].listunspent() + txid = utxo[0]['txid'] + value = utxo[0]['amount'] + vout = utxo[0]['vout'] + + send_value = satoshi_round((value - fee)/2) + inputs = [ {'txid' : txid, 'vout' : vout} ] + outputs = {} + for i in xrange(2): + outputs[self.nodes[0].getnewaddress()] = send_value + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) + signedtx = self.nodes[0].signrawtransaction(rawtx) + txid = self.nodes[0].sendrawtransaction(signedtx['hex']) + tx0_id = txid + value = send_value + + # Create tx1 + (tx1_id, tx1_value) = self.chain_transaction(self.nodes[0], tx0_id, 0, value, fee, 1) + + # Create tx2-7 + vout = 1 + txid = tx0_id + for i in xrange(6): + (txid, sent_value) = self.chain_transaction(self.nodes[0], txid, vout, value, fee, 1) + vout = 0 + value = sent_value + + # Mine these in a block + self.nodes[0].generate(1) + self.sync_all() + + # Now generate tx8, with a big fee + inputs = [ {'txid' : tx1_id, 'vout': 0}, {'txid' : txid, 'vout': 0} ] + outputs = { self.nodes[0].getnewaddress() : send_value + value - 4*fee } + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) + signedtx = self.nodes[0].signrawtransaction(rawtx) + txid = self.nodes[0].sendrawtransaction(signedtx['hex']) + sync_mempools(self.nodes) + + # Now try to disconnect the tip on each node... + self.nodes[1].invalidateblock(self.nodes[1].getbestblockhash()) + self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) + sync_blocks(self.nodes) + +if __name__ == '__main__': + MempoolPackagesTest().main() diff --git a/qa/rpc-tests/nodehandling.py b/qa/rpc-tests/nodehandling.py index d89cfcf59b..e383a3a12c 100755 --- a/qa/rpc-tests/nodehandling.py +++ b/qa/rpc-tests/nodehandling.py @@ -55,7 +55,7 @@ class NodeHandlingTest (BitcoinTestFramework): self.nodes[2].setban("192.168.0.1", "add", 1) #ban for 1 seconds self.nodes[2].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) #ban for 1000 seconds listBeforeShutdown = self.nodes[2].listbanned(); - assert_equal("192.168.0.1/255.255.255.255", listBeforeShutdown[2]['address']) #must be here + assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) #must be here time.sleep(2) #make 100% sure we expired 192.168.0.1 node time #stop node @@ -63,9 +63,9 @@ class NodeHandlingTest (BitcoinTestFramework): self.nodes[2] = start_node(2, self.options.tmpdir) listAfterShutdown = self.nodes[2].listbanned(); - assert_equal("127.0.0.0/255.255.255.0", listAfterShutdown[0]['address']) - assert_equal("127.0.0.0/255.255.255.255", listAfterShutdown[1]['address']) - assert_equal("2001:4000::/ffff:e000:0:0:0:0:0:0", listAfterShutdown[2]['address']) + assert_equal("127.0.0.0/24", listAfterShutdown[0]['address']) + assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) + assert_equal("/19" in listAfterShutdown[2]['address'], True) ########################### # RPC disconnectnode test # diff --git a/qa/rpc-tests/rest.py b/qa/rpc-tests/rest.py index 3a035f996c..e084ad55ab 100755 --- a/qa/rpc-tests/rest.py +++ b/qa/rpc-tests/rest.py @@ -32,10 +32,20 @@ def deser_uint256(f): r += t << (i * 32) return r -#allows simple http get calls with a request body -def http_get_call(host, port, path, requestdata = '', response_object = 0): +#allows simple http get calls +def http_get_call(host, port, path, response_object = 0): conn = httplib.HTTPConnection(host, port) - conn.request('GET', path, requestdata) + conn.request('GET', path) + + if response_object: + return conn.getresponse() + + return conn.getresponse().read() + +#allows simple http post calls with a request body +def http_post_call(host, port, path, requestdata = '', response_object = 0): + conn = httplib.HTTPConnection(host, port) + conn.request('POST', path, requestdata) if response_object: return conn.getresponse() @@ -137,7 +147,7 @@ class RESTTest (BitcoinTestFramework): binaryRequest += binascii.unhexlify(vintx); binaryRequest += pack("i", 0); - bin_response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', binaryRequest) + bin_response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', binaryRequest) output = StringIO.StringIO() output.write(bin_response) output.seek(0) @@ -175,14 +185,14 @@ class RESTTest (BitcoinTestFramework): #do some invalid requests json_request = '{"checkmempool' - response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True) + response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True) assert_equal(response.status, 500) #must be a 500 because we send a invalid json request json_request = '{"checkmempool' - response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', json_request, True) + response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', json_request, True) assert_equal(response.status, 500) #must be a 500 because we send a invalid bin request - response = http_get_call(url.hostname, url.port, '/rest/getutxos/checkmempool'+self.FORMAT_SEPARATOR+'bin', '', True) + response = http_post_call(url.hostname, url.port, '/rest/getutxos/checkmempool'+self.FORMAT_SEPARATOR+'bin', '', True) assert_equal(response.status, 500) #must be a 500 because we send a invalid bin request #test limits @@ -190,14 +200,14 @@ class RESTTest (BitcoinTestFramework): for x in range(0, 20): json_request += txid+'-'+str(n)+'/' json_request = json_request.rstrip("/") - response = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) + response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) assert_equal(response.status, 500) #must be a 500 because we exceeding the limits json_request = '/checkmempool/' for x in range(0, 15): json_request += txid+'-'+str(n)+'/' json_request = json_request.rstrip("/"); - response = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) + response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) assert_equal(response.status, 200) #must be a 500 because we exceeding the limits self.nodes[0].generate(1) #generate block to not affect upcoming tests @@ -208,27 +218,27 @@ class RESTTest (BitcoinTestFramework): ################ # check binary format - response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True) + response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", True) assert_equal(response.status, 200) assert_greater_than(int(response.getheader('content-length')), 80) response_str = response.read() # compare with block header - response_header = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True) + response_header = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"bin", True) assert_equal(response_header.status, 200) assert_equal(int(response_header.getheader('content-length')), 80) response_header_str = response_header.read() assert_equal(response_str[0:80], response_header_str) # check block hex format - response_hex = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"hex", "", True) + response_hex = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True) assert_equal(response_hex.status, 200) assert_greater_than(int(response_hex.getheader('content-length')), 160) response_hex_str = response_hex.read() assert_equal(response_str.encode("hex")[0:160], response_hex_str[0:160]) # compare with hex block header - response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", "", True) + response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True) assert_equal(response_header_hex.status, 200) assert_greater_than(int(response_header_hex.getheader('content-length')), 160) response_header_hex_str = response_header_hex.read() @@ -241,7 +251,7 @@ class RESTTest (BitcoinTestFramework): assert_equal(block_json_obj['hash'], bb_hash) # compare with json block header - response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"json", "", True) + response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"json", True) assert_equal(response_header_json.status, 200) response_header_json_str = response_header_json.read() json_obj = json.loads(response_header_json_str, parse_float=decimal.Decimal) @@ -265,7 +275,7 @@ class RESTTest (BitcoinTestFramework): #see if we can get 5 headers in one response self.nodes[1].generate(5) self.sync_all() - response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/5/'+bb_hash+self.FORMAT_SEPARATOR+"json", "", True) + response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/5/'+bb_hash+self.FORMAT_SEPARATOR+"json", True) assert_equal(response_header_json.status, 200) response_header_json_str = response_header_json.read() json_obj = json.loads(response_header_json_str) @@ -278,7 +288,7 @@ class RESTTest (BitcoinTestFramework): assert_equal(json_obj['txid'], tx_hash) # check hex format response - hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", "", True) + hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", True) assert_equal(hex_string.status, 200) assert_greater_than(int(response.getheader('content-length')), 10) diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index bc7d655fdf..33014dc139 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -106,6 +106,26 @@ class AuthServiceProxy(object): name = "%s.%s" % (self.__service_name, name) return AuthServiceProxy(self.__service_url, name, connection=self.__conn) + def _request(self, method, path, postdata): + ''' + Do a HTTP request, with retry if we get disconnected (e.g. due to a timeout). + This is a workaround for https://bugs.python.org/issue3566 which is fixed in Python 3.5. + ''' + headers = {'Host': self.__url.hostname, + 'User-Agent': USER_AGENT, + 'Authorization': self.__auth_header, + 'Content-type': 'application/json'} + try: + self.__conn.request(method, path, postdata, headers) + return self._get_response() + except httplib.BadStatusLine as e: + if e.line == "''": # if connection was closed, try again + self.__conn.close() + self.__conn.request(method, path, postdata, headers) + return self._get_response() + else: + raise + def __call__(self, *args): AuthServiceProxy.__id_count += 1 @@ -115,13 +135,7 @@ class AuthServiceProxy(object): 'method': self.__service_name, 'params': args, 'id': AuthServiceProxy.__id_count}, default=EncodeDecimal) - self.__conn.request('POST', self.__url.path, postdata, - {'Host': self.__url.hostname, - 'User-Agent': USER_AGENT, - 'Authorization': self.__auth_header, - 'Content-type': 'application/json'}) - - response = self._get_response() + response = self._request('POST', self.__url.path, postdata) if response['error'] is not None: raise JSONRPCException(response['error']) elif 'result' not in response: @@ -133,13 +147,7 @@ class AuthServiceProxy(object): def _batch(self, rpc_call_list): postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal) log.debug("--> "+postdata) - self.__conn.request('POST', self.__url.path, postdata, - {'Host': self.__url.hostname, - 'User-Agent': USER_AGENT, - 'Authorization': self.__auth_header, - 'Content-type': 'application/json'}) - - return self._get_response() + return self._request('POST', self.__url.path, postdata) def _get_response(self): http_response = self.__conn.getresponse() diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index c236ec2602..3759cc8162 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -78,8 +78,17 @@ def initialize_chain(test_dir): bitcoind and bitcoin-cli must be in search path. """ - if not os.path.isdir(os.path.join("cache", "node0")): - devnull = open("/dev/null", "w+") + if (not os.path.isdir(os.path.join("cache","node0")) + or not os.path.isdir(os.path.join("cache","node1")) + or not os.path.isdir(os.path.join("cache","node2")) + or not os.path.isdir(os.path.join("cache","node3"))): + + #find and delete old cache directories if any exist + for i in range(4): + if os.path.isdir(os.path.join("cache","node"+str(i))): + shutil.rmtree(os.path.join("cache","node"+str(i))) + + devnull = open(os.devnull, "w") # Create cache directories, run bitcoinds: for i in range(4): datadir=initialize_datadir("cache", i) @@ -171,7 +180,7 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= args = [ binary, "-datadir="+datadir, "-keypool=1", "-discover=0", "-rest" ] if extra_args is not None: args.extend(extra_args) bitcoind_processes[i] = subprocess.Popen(args) - devnull = open("/dev/null", "w+") + devnull = open(os.devnull, "w") if os.getenv("PYTHON_DEBUG", ""): print "start_node: bitcoind started, calling bitcoin-cli -rpcwait getblockcount" subprocess.check_call([ os.getenv("BITCOINCLI", "bitcoin-cli"), "-datadir="+datadir] + diff --git a/qa/rpc-tests/zmq_test.py b/qa/rpc-tests/zmq_test.py new file mode 100755 index 0000000000..fffaf677d6 --- /dev/null +++ b/qa/rpc-tests/zmq_test.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python2 +# Copyright (c) 2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test ZMQ interface +# + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +import zmq +import binascii +from test_framework.mininode import hash256 + +try: + import http.client as httplib +except ImportError: + import httplib +try: + import urllib.parse as urlparse +except ImportError: + import urlparse + +class ZMQTest (BitcoinTestFramework): + + port = 28332 + + def setup_nodes(self): + self.zmqContext = zmq.Context() + self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) + self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashblock") + self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, "hashtx") + self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % self.port) + # Note: proxies are not used to connect to local nodes + # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost + return start_nodes(4, self.options.tmpdir, extra_args=[ + ['-zmqpubhashtx=tcp://127.0.0.1:'+str(self.port), '-zmqpubhashblock=tcp://127.0.0.1:'+str(self.port)], + [], + [], + [] + ]) + + def run_test(self): + self.sync_all() + + genhashes = self.nodes[0].generate(1); + self.sync_all() + + print "listen..." + msg = self.zmqSubSocket.recv_multipart() + topic = str(msg[0]) + body = msg[1] + + msg = self.zmqSubSocket.recv_multipart() + topic = str(msg[0]) + body = msg[1] + blkhash = binascii.hexlify(body) + + assert_equal(genhashes[0], blkhash) #blockhash from generate must be equal to the hash received over zmq + + n = 10 + genhashes = self.nodes[1].generate(n); + self.sync_all() + + zmqHashes = [] + for x in range(0,n*2): + msg = self.zmqSubSocket.recv_multipart() + topic = str(msg[0]) + body = msg[1] + if topic == "hashblock": + zmqHashes.append(binascii.hexlify(body)) + + for x in range(0,n): + assert_equal(genhashes[x], zmqHashes[x]) #blockhash from generate must be equal to the hash received over zmq + + #test tx from a second node + hashRPC = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) + self.sync_all() + + #now we should receive a zmq msg because the tx was broadcastet + msg = self.zmqSubSocket.recv_multipart() + topic = str(msg[0]) + body = msg[1] + hashZMQ = "" + if topic == "hashtx": + hashZMQ = binascii.hexlify(body) + + assert_equal(hashRPC, hashZMQ) #blockhash from generate must be equal to the hash received over zmq + + +if __name__ == '__main__': + ZMQTest ().main () diff --git a/share/pixmaps/addressbook16.bmp b/share/pixmaps/addressbook16.bmp Binary files differdeleted file mode 100644 index c5576910b1..0000000000 --- a/share/pixmaps/addressbook16.bmp +++ /dev/null diff --git a/share/pixmaps/addressbook16mask.bmp b/share/pixmaps/addressbook16mask.bmp Binary files differdeleted file mode 100644 index d3a478d1ad..0000000000 --- a/share/pixmaps/addressbook16mask.bmp +++ /dev/null diff --git a/share/pixmaps/addressbook20.bmp b/share/pixmaps/addressbook20.bmp Binary files differdeleted file mode 100644 index 2b33b228aa..0000000000 --- a/share/pixmaps/addressbook20.bmp +++ /dev/null diff --git a/share/pixmaps/addressbook20mask.bmp b/share/pixmaps/addressbook20mask.bmp Binary files differdeleted file mode 100644 index 56ce6125db..0000000000 --- a/share/pixmaps/addressbook20mask.bmp +++ /dev/null diff --git a/share/pixmaps/bitcoin-bc.ico b/share/pixmaps/bitcoin-bc.ico Binary files differdeleted file mode 100644 index 88cc240e2d..0000000000 --- a/share/pixmaps/bitcoin-bc.ico +++ /dev/null diff --git a/share/pixmaps/check.ico b/share/pixmaps/check.ico Binary files differdeleted file mode 100644 index 0c4e6e8147..0000000000 --- a/share/pixmaps/check.ico +++ /dev/null diff --git a/share/pixmaps/favicon.ico b/share/pixmaps/favicon.ico Binary files differdeleted file mode 100644 index 754eebc488..0000000000 --- a/share/pixmaps/favicon.ico +++ /dev/null diff --git a/share/pixmaps/send16.bmp b/share/pixmaps/send16.bmp Binary files differdeleted file mode 100644 index 676b5c4b49..0000000000 --- a/share/pixmaps/send16.bmp +++ /dev/null diff --git a/share/pixmaps/send16mask.bmp b/share/pixmaps/send16mask.bmp Binary files differdeleted file mode 100644 index 06c747f934..0000000000 --- a/share/pixmaps/send16mask.bmp +++ /dev/null diff --git a/share/pixmaps/send16masknoshadow.bmp b/share/pixmaps/send16masknoshadow.bmp Binary files differdeleted file mode 100644 index faf24e0d8a..0000000000 --- a/share/pixmaps/send16masknoshadow.bmp +++ /dev/null diff --git a/share/pixmaps/send20.bmp b/share/pixmaps/send20.bmp Binary files differdeleted file mode 100644 index 2b90422b38..0000000000 --- a/share/pixmaps/send20.bmp +++ /dev/null diff --git a/share/pixmaps/send20mask.bmp b/share/pixmaps/send20mask.bmp Binary files differdeleted file mode 100644 index f124d0da08..0000000000 --- a/share/pixmaps/send20mask.bmp +++ /dev/null diff --git a/share/qt/img/reload.png b/share/qt/img/reload.png Binary files differdeleted file mode 100644 index 9068db9a63..0000000000 --- a/share/qt/img/reload.png +++ /dev/null diff --git a/share/qt/img/reload.xcf b/share/qt/img/reload.xcf Binary files differdeleted file mode 100644 index dc8be62831..0000000000 --- a/share/qt/img/reload.xcf +++ /dev/null diff --git a/share/qt/make_spinner.py b/share/qt/make_spinner.py deleted file mode 100755 index bb19e91508..0000000000 --- a/share/qt/make_spinner.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -# W.J. van der Laan, 2011 -# Make spinning animation from a .png -# Requires imagemagick 6.7+ -from __future__ import division -from os import path -from PIL import Image -from subprocess import Popen - -SRC='img/reload.png' -TMPDIR='../../src/qt/res/movies/' -TMPNAME='spinner-%03i.png' -NUMFRAMES=35 -FRAMERATE=10.0 -CONVERT='convert' -CLOCKWISE=True -DSIZE=(16,16) - -im_src = Image.open(SRC) - -if CLOCKWISE: - im_src = im_src.transpose(Image.FLIP_LEFT_RIGHT) - -def frame_to_filename(frame): - return path.join(TMPDIR, TMPNAME % frame) - -frame_files = [] -for frame in xrange(NUMFRAMES): - rotation = (frame + 0.5) / NUMFRAMES * 360.0 - if CLOCKWISE: - rotation = -rotation - im_new = im_src.rotate(rotation, Image.BICUBIC) - im_new.thumbnail(DSIZE, Image.ANTIALIAS) - outfile = frame_to_filename(frame) - im_new.save(outfile, 'png') - frame_files.append(outfile) - - diff --git a/share/ui.rc b/share/ui.rc deleted file mode 100644 index 063641cba2..0000000000 --- a/share/ui.rc +++ /dev/null @@ -1,15 +0,0 @@ -bitcoin ICON "pixmaps/bitcoin.ico"
-
-#include "wx/msw/wx.rc"
-
-check ICON "pixmaps/check.ico"
-send16 BITMAP "pixmaps/send16.bmp"
-send16mask BITMAP "pixmaps/send16mask.bmp"
-send16masknoshadow BITMAP "pixmaps/send16masknoshadow.bmp"
-send20 BITMAP "pixmaps/send20.bmp"
-send20mask BITMAP "pixmaps/send20mask.bmp"
-addressbook16 BITMAP "pixmaps/addressbook16.bmp"
-addressbook16mask BITMAP "pixmaps/addressbook16mask.bmp"
-addressbook20 BITMAP "pixmaps/addressbook20.bmp"
-addressbook20mask BITMAP "pixmaps/addressbook20mask.bmp"
-favicon ICON "pixmaps/favicon.ico"
diff --git a/src/Makefile.am b/src/Makefile.am index cc8dded413..67e848be39 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -48,6 +48,9 @@ if ENABLE_WALLET BITCOIN_INCLUDES += $(BDB_CPPFLAGS) EXTRA_LIBRARIES += libbitcoin_wallet.a endif +if ENABLE_ZMQ +EXTRA_LIBRARIES += libbitcoin_zmq.a +endif if BUILD_BITCOIN_LIBS lib_LTLIBRARIES = libbitcoinconsensus.la @@ -98,6 +101,8 @@ BITCOIN_CORE_H = \ eccryptoverify.h \ ecwrapper.h \ hash.h \ + httprpc.h \ + httpserver.h \ init.h \ key.h \ keystore.h \ @@ -119,6 +124,7 @@ BITCOIN_CORE_H = \ protocol.h \ pubkey.h \ random.h \ + reverselock.h \ rpcclient.h \ rpcprotocol.h \ rpcserver.h \ @@ -154,7 +160,12 @@ BITCOIN_CORE_H = \ wallet/db.h \ wallet/wallet.h \ wallet/wallet_ismine.h \ - wallet/walletdb.h + wallet/walletdb.h \ + zmq/zmqabstractnotifier.h \ + zmq/zmqconfig.h\ + zmq/zmqnotificationinterface.h \ + zmq/zmqpublishnotifier.h + obj/build.h: FORCE @$(MKDIR_P) $(builddir)/obj @@ -163,13 +174,15 @@ obj/build.h: FORCE libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # server: shared between bitcoind and bitcoin-qt -libbitcoin_server_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) +libbitcoin_server_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_SOURCES = \ addrman.cpp \ alert.cpp \ bloom.cpp \ chain.cpp \ checkpoints.cpp \ + httprpc.cpp \ + httpserver.cpp \ init.cpp \ leveldbwrapper.cpp \ main.cpp \ @@ -194,6 +207,17 @@ libbitcoin_server_a_SOURCES = \ validationinterface.cpp \ $(BITCOIN_CORE_H) +if ENABLE_ZMQ +LIBBITCOIN_ZMQ=libbitcoin_zmq.a + +libbitcoin_zmq_a_CPPFLAGS = $(BITCOIN_INCLUDES) +libbitcoin_zmq_a_SOURCES = \ + zmq/zmqabstractnotifier.cpp \ + zmq/zmqnotificationinterface.cpp \ + zmq/zmqpublishnotifier.cpp +endif + + # wallet: shared between bitcoind and bitcoin-qt, but only linked # when wallet enabled libbitcoin_wallet_a_CPPFLAGS = $(BITCOIN_INCLUDES) @@ -315,16 +339,19 @@ bitcoind_LDADD = \ $(LIBMEMENV) \ $(LIBSECP256K1) +if ENABLE_ZMQ +bitcoind_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +endif + if ENABLE_WALLET bitcoind_LDADD += libbitcoin_wallet.a endif -bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) -# +bitcoind_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) # bitcoin-cli binary # bitcoin_cli_SOURCES = bitcoin-cli.cpp -bitcoin_cli_CPPFLAGS = $(BITCOIN_INCLUDES) +bitcoin_cli_CPPFLAGS = $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) bitcoin_cli_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) if TARGET_WINDOWS @@ -337,7 +364,7 @@ bitcoin_cli_LDADD = \ $(LIBBITCOIN_UTIL) \ $(LIBSECP256K1) -bitcoin_cli_LDADD += $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) +bitcoin_cli_LDADD += $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) # # bitcoin-tx binary # diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 748f2b14d5..3330ed2890 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -87,7 +87,7 @@ QT_FORMS_UI = \ qt/forms/overviewpage.ui \ qt/forms/receivecoinsdialog.ui \ qt/forms/receiverequestdialog.ui \ - qt/forms/rpcconsole.ui \ + qt/forms/debugwindow.ui \ qt/forms/sendcoinsdialog.ui \ qt/forms/sendcoinsentry.ui \ qt/forms/signverifymessagedialog.ui \ @@ -97,6 +97,7 @@ QT_MOC_CPP = \ qt/moc_addressbookpage.cpp \ qt/moc_addresstablemodel.cpp \ qt/moc_askpassphrasedialog.cpp \ + qt/moc_bantablemodel.cpp \ qt/moc_bitcoinaddressvalidator.cpp \ qt/moc_bitcoinamountfield.cpp \ qt/moc_bitcoingui.cpp \ @@ -162,6 +163,7 @@ BITCOIN_QT_H = \ qt/addressbookpage.h \ qt/addresstablemodel.h \ qt/askpassphrasedialog.h \ + qt/bantablemodel.h \ qt/bitcoinaddressvalidator.h \ qt/bitcoinamountfield.h \ qt/bitcoingui.h \ @@ -260,6 +262,7 @@ RES_ICONS = \ qt/res/icons/verify.png BITCOIN_QT_CPP = \ + qt/bantablemodel.cpp \ qt/bitcoinaddressvalidator.cpp \ qt/bitcoinamountfield.cpp \ qt/bitcoingui.cpp \ @@ -361,8 +364,12 @@ qt_bitcoin_qt_LDADD = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) if ENABLE_WALLET qt_bitcoin_qt_LDADD += $(LIBBITCOIN_WALLET) endif +if ENABLE_ZMQ +qt_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +endif qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_UNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ - $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) + $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ + $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) qt_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_bitcoin_qt_LIBTOOLFLAGS = --tag CXX diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index c5392cf307..6554580bea 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -30,9 +30,13 @@ qt_test_test_bitcoin_qt_LDADD = $(LIBBITCOINQT) $(LIBBITCOIN_SERVER) if ENABLE_WALLET qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_WALLET) endif +if ENABLE_ZMQ +qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_UNIVALUE) $(LIBLEVELDB) \ $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ - $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) + $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ + $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) CLEAN_BITCOIN_QT_TEST = $(TEST_QT_MOC_CPP) qt/test/*.gcda qt/test/*.gcno diff --git a/src/Makefile.test.include b/src/Makefile.test.include index fc4e047c35..cee35926a5 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -62,6 +62,7 @@ BITCOIN_TESTS =\ test/pmt_tests.cpp \ test/policyestimator_tests.cpp \ test/pow_tests.cpp \ + test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ @@ -99,6 +100,10 @@ endif test_test_bitcoin_LDADD += $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static +if ENABLE_ZMQ +test_test_bitcoin_LDADD += $(ZMQ_LIBS) +endif + nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) $(BITCOIN_TESTS): $(GENERATED_TEST_FILES) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 1c5a312874..7839b3b6b4 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -11,11 +11,19 @@ #include "utilstrencodings.h" #include <boost/filesystem/operations.hpp> +#include <stdio.h> + +#include <event2/event.h> +#include <event2/http.h> +#include <event2/buffer.h> +#include <event2/keyvalq_struct.h> #include "univalue/univalue.h" using namespace std; +static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; + std::string HelpMessageCli() { string strUsage; @@ -31,9 +39,7 @@ std::string HelpMessageCli() strUsage += HelpMessageOpt("-rpcwait", _("Wait for RPC server to start")); strUsage += HelpMessageOpt("-rpcuser=<user>", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections")); - - strUsage += HelpMessageGroup(_("SSL options: (see the Bitcoin Wiki for SSL setup instructions)")); - strUsage += HelpMessageOpt("-rpcssl", _("Use OpenSSL (https) for JSON-RPC connections")); + strUsage += HelpMessageOpt("-rpcclienttimeout=<n>", strprintf(_("Timeout during HTTP requests (default: %d)"), DEFAULT_HTTP_CLIENT_TIMEOUT)); return strUsage; } @@ -92,32 +98,75 @@ static bool AppInitRPC(int argc, char* argv[]) fprintf(stderr, "Error: Invalid combination of -regtest and -testnet.\n"); return false; } + if (GetBoolArg("-rpcssl", false)) + { + fprintf(stderr, "Error: SSL mode for RPC (-rpcssl) is no longer supported.\n"); + return false; + } return true; } -UniValue CallRPC(const string& strMethod, const UniValue& params) + +/** Reply structure for request_done to fill in */ +struct HTTPReply { - // Connect to localhost - bool fUseSSL = GetBoolArg("-rpcssl", false); - boost::asio::io_service io_service; - boost::asio::ssl::context context(io_service, boost::asio::ssl::context::sslv23); - context.set_options(boost::asio::ssl::context::no_sslv2 | boost::asio::ssl::context::no_sslv3); - boost::asio::ssl::stream<boost::asio::ip::tcp::socket> sslStream(io_service, context); - SSLIOStreamDevice<boost::asio::ip::tcp> d(sslStream, fUseSSL); - boost::iostreams::stream< SSLIOStreamDevice<boost::asio::ip::tcp> > stream(d); - - const bool fConnected = d.connect(GetArg("-rpcconnect", "127.0.0.1"), GetArg("-rpcport", itostr(BaseParams().RPCPort()))); - if (!fConnected) - throw CConnectionFailed("couldn't connect to server"); + int status; + std::string body; +}; + +static void http_request_done(struct evhttp_request *req, void *ctx) +{ + HTTPReply *reply = static_cast<HTTPReply*>(ctx); + + if (req == NULL) { + /* If req is NULL, it means an error occurred while connecting, but + * I'm not sure how to find out which one. We also don't really care. + */ + reply->status = 0; + return; + } - // Find credentials to use + reply->status = evhttp_request_get_response_code(req); + + struct evbuffer *buf = evhttp_request_get_input_buffer(req); + if (buf) + { + size_t size = evbuffer_get_length(buf); + const char *data = (const char*)evbuffer_pullup(buf, size); + if (data) + reply->body = std::string(data, size); + evbuffer_drain(buf, size); + } +} + +UniValue CallRPC(const string& strMethod, const UniValue& params) +{ + std::string host = GetArg("-rpcconnect", "127.0.0.1"); + int port = GetArg("-rpcport", BaseParams().RPCPort()); + + // Create event base + struct event_base *base = event_base_new(); // TODO RAII + if (!base) + throw runtime_error("cannot create event_base"); + + // Synchronously look up hostname + struct evhttp_connection *evcon = evhttp_connection_base_new(base, NULL, host.c_str(), port); // TODO RAII + if (evcon == NULL) + throw runtime_error("create connection failed"); + evhttp_connection_set_timeout(evcon, GetArg("-rpcclienttimeout", DEFAULT_HTTP_CLIENT_TIMEOUT)); + + HTTPReply response; + struct evhttp_request *req = evhttp_request_new(http_request_done, (void*)&response); // TODO RAII + if (req == NULL) + throw runtime_error("create http request failed"); + + // Get credentials std::string strRPCUserColonPass; if (mapArgs["-rpcpassword"] == "") { // Try fall back to cookie-based authentication if no password is provided if (!GetAuthCookie(&strRPCUserColonPass)) { throw runtime_error(strprintf( - _("You must set rpcpassword=<password> in the configuration file:\n%s\n" - "If the file does not exist, create it with owner-readable-only file permissions."), + _("Could not locate RPC credentials. No authentication cookie could be found, and no rpcpassword is set in the configuration file (%s)"), GetConfigFile().string().c_str())); } @@ -125,34 +174,41 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]; } - // HTTP basic authentication - map<string, string> mapRequestHeaders; - mapRequestHeaders["Authorization"] = string("Basic ") + EncodeBase64(strRPCUserColonPass); - - // Send request - string strRequest = JSONRPCRequest(strMethod, params, 1); - string strPost = HTTPPost(strRequest, mapRequestHeaders); - stream << strPost << std::flush; - - // Receive HTTP reply status - int nProto = 0; - int nStatus = ReadHTTPStatus(stream, nProto); + struct evkeyvalq *output_headers = evhttp_request_get_output_headers(req); + assert(output_headers); + evhttp_add_header(output_headers, "Host", host.c_str()); + evhttp_add_header(output_headers, "Connection", "close"); + evhttp_add_header(output_headers, "Authorization", (std::string("Basic ") + EncodeBase64(strRPCUserColonPass)).c_str()); + + // Attach request data + std::string strRequest = JSONRPCRequest(strMethod, params, 1); + struct evbuffer * output_buffer = evhttp_request_get_output_buffer(req); + assert(output_buffer); + evbuffer_add(output_buffer, strRequest.data(), strRequest.size()); + + int r = evhttp_make_request(evcon, req, EVHTTP_REQ_POST, "/"); + if (r != 0) { + evhttp_connection_free(evcon); + event_base_free(base); + throw CConnectionFailed("send http request failed"); + } - // Receive HTTP reply message headers and body - map<string, string> mapHeaders; - string strReply; - ReadHTTPMessage(stream, mapHeaders, strReply, nProto, std::numeric_limits<size_t>::max()); + event_base_dispatch(base); + evhttp_connection_free(evcon); + event_base_free(base); - if (nStatus == HTTP_UNAUTHORIZED) + if (response.status == 0) + throw CConnectionFailed("couldn't connect to server"); + else if (response.status == HTTP_UNAUTHORIZED) throw runtime_error("incorrect rpcuser or rpcpassword (authorization failed)"); - else if (nStatus >= 400 && nStatus != HTTP_BAD_REQUEST && nStatus != HTTP_NOT_FOUND && nStatus != HTTP_INTERNAL_SERVER_ERROR) - throw runtime_error(strprintf("server returned HTTP error %d", nStatus)); - else if (strReply.empty()) + else if (response.status >= 400 && response.status != HTTP_BAD_REQUEST && response.status != HTTP_NOT_FOUND && response.status != HTTP_INTERNAL_SERVER_ERROR) + throw runtime_error(strprintf("server returned HTTP error %d", response.status)); + else if (response.body.empty()) throw runtime_error("no response from server"); // Parse reply UniValue valReply(UniValue::VSTR); - if (!valReply.read(strReply)) + if (!valReply.read(response.body)) throw runtime_error("couldn't parse reply from server"); const UniValue& reply = valReply.get_obj(); if (reply.empty()) @@ -248,6 +304,10 @@ int CommandLineRPC(int argc, char *argv[]) int main(int argc, char* argv[]) { SetupEnvironment(); + if (!SetupNetworking()) { + fprintf(stderr, "Error: Initializing networking failed\n"); + exit(1); + } try { if(!AppInitRPC(argc, argv)) diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 39bb301f44..b512f74c22 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -10,11 +10,16 @@ #include "noui.h" #include "scheduler.h" #include "util.h" +#include "httpserver.h" +#include "httprpc.h" +#include "rpcserver.h" #include <boost/algorithm/string/predicate.hpp> #include <boost/filesystem.hpp> #include <boost/thread.hpp> +#include <stdio.h> + /* Introduction text for doxygen: */ /*! \mainpage Developer documentation @@ -44,7 +49,7 @@ void WaitForShutdown(boost::thread_group* threadGroup) } if (threadGroup) { - threadGroup->interrupt_all(); + Interrupt(*threadGroup); threadGroup->join_all(); } } @@ -154,7 +159,7 @@ bool AppInit(int argc, char* argv[]) if (!fRet) { - threadGroup.interrupt_all(); + Interrupt(threadGroup); // threadGroup.join_all(); was left out intentionally here, because we didn't re-test all of // the startup-failure cases to make sure they don't result in a hang due to some // thread-blocking-waiting-for-another-thread-during-startup case diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 9c843f6e7e..15b86cdda6 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -31,7 +31,7 @@ static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesi genesis.nVersion = nVersion; genesis.vtx.push_back(txNew); genesis.hashPrevBlock.SetNull(); - genesis.hashMerkleRoot = genesis.BuildMerkleTree(); + genesis.hashMerkleRoot = genesis.ComputeMerkleRoot(); return genesis; } diff --git a/src/core_memusage.h b/src/core_memusage.h index 711135bb44..a05f59ee0c 100644 --- a/src/core_memusage.h +++ b/src/core_memusage.h @@ -48,7 +48,7 @@ static inline size_t RecursiveDynamicUsage(const CMutableTransaction& tx) { } static inline size_t RecursiveDynamicUsage(const CBlock& block) { - size_t mem = memusage::DynamicUsage(block.vtx) + memusage::DynamicUsage(block.vMerkleTree); + size_t mem = memusage::DynamicUsage(block.vtx); for (std::vector<CTransaction>::const_iterator it = block.vtx.begin(); it != block.vtx.end(); it++) { mem += RecursiveDynamicUsage(*it); } diff --git a/src/httprpc.cpp b/src/httprpc.cpp new file mode 100644 index 0000000000..98ac750bb1 --- /dev/null +++ b/src/httprpc.cpp @@ -0,0 +1,193 @@ +#include "httprpc.h" + +#include "base58.h" +#include "chainparams.h" +#include "httpserver.h" +#include "rpcprotocol.h" +#include "rpcserver.h" +#include "random.h" +#include "sync.h" +#include "util.h" +#include "utilstrencodings.h" +#include "ui_interface.h" + +#include <boost/algorithm/string.hpp> // boost::trim + +/** Simple one-shot callback timer to be used by the RPC mechanism to e.g. + * re-lock the wellet. + */ +class HTTPRPCTimer : public RPCTimerBase +{ +public: + HTTPRPCTimer(struct event_base* eventBase, boost::function<void(void)>& func, int64_t millis) : + ev(eventBase, false, func) + { + struct timeval tv; + tv.tv_sec = millis/1000; + tv.tv_usec = (millis%1000)*1000; + ev.trigger(&tv); + } +private: + HTTPEvent ev; +}; + +class HTTPRPCTimerInterface : public RPCTimerInterface +{ +public: + HTTPRPCTimerInterface(struct event_base* base) : base(base) + { + } + const char* Name() + { + return "HTTP"; + } + RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) + { + return new HTTPRPCTimer(base, func, millis); + } +private: + struct event_base* base; +}; + + +/* Pre-base64-encoded authentication token */ +static std::string strRPCUserColonPass; +/* Stored RPC timer interface (for unregistration) */ +static HTTPRPCTimerInterface* httpRPCTimerInterface = 0; + +static void JSONErrorReply(HTTPRequest* req, const UniValue& objError, const UniValue& id) +{ + // Send error reply from json-rpc error object + int nStatus = HTTP_INTERNAL_SERVER_ERROR; + int code = find_value(objError, "code").get_int(); + + if (code == RPC_INVALID_REQUEST) + nStatus = HTTP_BAD_REQUEST; + else if (code == RPC_METHOD_NOT_FOUND) + nStatus = HTTP_NOT_FOUND; + + std::string strReply = JSONRPCReply(NullUniValue, objError, id); + + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(nStatus, strReply); +} + +static bool RPCAuthorized(const std::string& strAuth) +{ + if (strRPCUserColonPass.empty()) // Belt-and-suspenders measure if InitRPCAuthentication was not called + return false; + if (strAuth.substr(0, 6) != "Basic ") + return false; + std::string strUserPass64 = strAuth.substr(6); + boost::trim(strUserPass64); + std::string strUserPass = DecodeBase64(strUserPass64); + return TimingResistantEqual(strUserPass, strRPCUserColonPass); +} + +static bool HTTPReq_JSONRPC(HTTPRequest* req, const std::string &) +{ + // JSONRPC handles only POST + if (req->GetRequestMethod() != HTTPRequest::POST) { + req->WriteReply(HTTP_BAD_METHOD, "JSONRPC server handles only POST requests"); + return false; + } + // Check authorization + std::pair<bool, std::string> authHeader = req->GetHeader("authorization"); + if (!authHeader.first) { + req->WriteReply(HTTP_UNAUTHORIZED); + return false; + } + + if (!RPCAuthorized(authHeader.second)) { + LogPrintf("ThreadRPCServer incorrect password attempt from %s\n", req->GetPeer().ToString()); + + /* Deter brute-forcing + If this results in a DoS the user really + shouldn't have their RPC port exposed. */ + MilliSleep(250); + + req->WriteReply(HTTP_UNAUTHORIZED); + return false; + } + + JSONRequest jreq; + try { + // Parse request + UniValue valRequest; + if (!valRequest.read(req->ReadBody())) + throw JSONRPCError(RPC_PARSE_ERROR, "Parse error"); + + std::string strReply; + // singleton request + if (valRequest.isObject()) { + jreq.parse(valRequest); + + UniValue result = tableRPC.execute(jreq.strMethod, jreq.params); + + // Send reply + strReply = JSONRPCReply(result, NullUniValue, jreq.id); + + // array of requests + } else if (valRequest.isArray()) + strReply = JSONRPCExecBatch(valRequest.get_array()); + else + throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error"); + + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strReply); + } catch (const UniValue& objError) { + JSONErrorReply(req, objError, jreq.id); + return false; + } catch (const std::exception& e) { + JSONErrorReply(req, JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); + return false; + } + return true; +} + +static bool InitRPCAuthentication() +{ + if (mapArgs["-rpcpassword"] == "") + { + LogPrintf("No rpcpassword set - using random cookie authentication\n"); + if (!GenerateAuthCookie(&strRPCUserColonPass)) { + uiInterface.ThreadSafeMessageBox( + _("Error: A fatal internal error occurred, see debug.log for details"), // Same message as AbortNode + "", CClientUIInterface::MSG_ERROR); + return false; + } + } else { + strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]; + } + return true; +} + +bool StartHTTPRPC() +{ + LogPrint("rpc", "Starting HTTP RPC server\n"); + if (!InitRPCAuthentication()) + return false; + + RegisterHTTPHandler("/", true, HTTPReq_JSONRPC); + + assert(EventBase()); + httpRPCTimerInterface = new HTTPRPCTimerInterface(EventBase()); + RPCRegisterTimerInterface(httpRPCTimerInterface); + return true; +} + +void InterruptHTTPRPC() +{ + LogPrint("rpc", "Interrupting HTTP RPC server\n"); +} + +void StopHTTPRPC() +{ + LogPrint("rpc", "Stopping HTTP RPC server\n"); + UnregisterHTTPHandler("/", true); + if (httpRPCTimerInterface) { + RPCUnregisterTimerInterface(httpRPCTimerInterface); + delete httpRPCTimerInterface; + httpRPCTimerInterface = 0; + } +} diff --git a/src/httprpc.h b/src/httprpc.h new file mode 100644 index 0000000000..d354457188 --- /dev/null +++ b/src/httprpc.h @@ -0,0 +1,37 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_HTTPRPC_H +#define BITCOIN_HTTPRPC_H + +#include <string> +#include <map> + +class HTTPRequest; + +/** Start HTTP RPC subsystem. + * Precondition; HTTP and RPC has been started. + */ +bool StartHTTPRPC(); +/** Interrupt HTTP RPC subsystem. + */ +void InterruptHTTPRPC(); +/** Stop HTTP RPC subsystem. + * Precondition; HTTP and RPC has been stopped. + */ +void StopHTTPRPC(); + +/** Start HTTP REST subsystem. + * Precondition; HTTP and RPC has been started. + */ +bool StartREST(); +/** Interrupt RPC REST subsystem. + */ +void InterruptREST(); +/** Stop HTTP REST subsystem. + * Precondition; HTTP and RPC has been stopped. + */ +void StopREST(); + +#endif diff --git a/src/httpserver.cpp b/src/httpserver.cpp new file mode 100644 index 0000000000..600e57b7cc --- /dev/null +++ b/src/httpserver.cpp @@ -0,0 +1,594 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "httpserver.h" + +#include "chainparamsbase.h" +#include "compat.h" +#include "util.h" +#include "netbase.h" +#include "rpcprotocol.h" // For HTTP status codes +#include "sync.h" +#include "ui_interface.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <sys/types.h> +#include <sys/stat.h> +#include <signal.h> + +#include <event2/event.h> +#include <event2/http.h> +#include <event2/thread.h> +#include <event2/buffer.h> +#include <event2/util.h> +#include <event2/keyvalq_struct.h> + +#ifdef EVENT__HAVE_NETINET_IN_H +#include <netinet/in.h> +#ifdef _XOPEN_SOURCE_EXTENDED +#include <arpa/inet.h> +#endif +#endif + +#include <boost/algorithm/string/case_conv.hpp> // for to_lower() +#include <boost/foreach.hpp> +#include <boost/scoped_ptr.hpp> + +/** HTTP request work item */ +class HTTPWorkItem : public HTTPClosure +{ +public: + HTTPWorkItem(HTTPRequest* req, const std::string &path, const HTTPRequestHandler& func): + req(req), path(path), func(func) + { + } + void operator()() + { + func(req.get(), path); + } + + boost::scoped_ptr<HTTPRequest> req; + +private: + std::string path; + HTTPRequestHandler func; +}; + +/** Simple work queue for distributing work over multiple threads. + * Work items are simply callable objects. + */ +template <typename WorkItem> +class WorkQueue +{ +private: + /** Mutex protects entire object */ + CWaitableCriticalSection cs; + CConditionVariable cond; + /* XXX in C++11 we can use std::unique_ptr here and avoid manual cleanup */ + std::deque<WorkItem*> queue; + bool running; + size_t maxDepth; + +public: + WorkQueue(size_t maxDepth) : running(true), + maxDepth(maxDepth) + { + } + /* Precondition: worker threads have all stopped */ + ~WorkQueue() + { + while (!queue.empty()) { + delete queue.front(); + queue.pop_front(); + } + } + /** Enqueue a work item */ + bool Enqueue(WorkItem* item) + { + boost::unique_lock<boost::mutex> lock(cs); + if (queue.size() >= maxDepth) { + return false; + } + queue.push_back(item); + cond.notify_one(); + return true; + } + /** Thread function */ + void Run() + { + while (running) { + WorkItem* i = 0; + { + boost::unique_lock<boost::mutex> lock(cs); + while (running && queue.empty()) + cond.wait(lock); + if (!running) + break; + i = queue.front(); + queue.pop_front(); + } + (*i)(); + delete i; + } + } + /** Interrupt and exit loops */ + void Interrupt() + { + boost::unique_lock<boost::mutex> lock(cs); + running = false; + cond.notify_all(); + } + + /** Return current depth of queue */ + size_t Depth() + { + boost::unique_lock<boost::mutex> lock(cs); + return queue.size(); + } +}; + +struct HTTPPathHandler +{ + HTTPPathHandler() {} + HTTPPathHandler(std::string prefix, bool exactMatch, HTTPRequestHandler handler): + prefix(prefix), exactMatch(exactMatch), handler(handler) + { + } + std::string prefix; + bool exactMatch; + HTTPRequestHandler handler; +}; + +/** HTTP module state */ + +//! libevent event loop +static struct event_base* eventBase = 0; +//! HTTP server +struct evhttp* eventHTTP = 0; +//! List of subnets to allow RPC connections from +static std::vector<CSubNet> rpc_allow_subnets; +//! Work queue for handling longer requests off the event loop thread +static WorkQueue<HTTPClosure>* workQueue = 0; +//! Handlers for (sub)paths +std::vector<HTTPPathHandler> pathHandlers; + +/** Check if a network address is allowed to access the HTTP server */ +static bool ClientAllowed(const CNetAddr& netaddr) +{ + if (!netaddr.IsValid()) + return false; + BOOST_FOREACH (const CSubNet& subnet, rpc_allow_subnets) + if (subnet.Match(netaddr)) + return true; + return false; +} + +/** Initialize ACL list for HTTP server */ +static bool InitHTTPAllowList() +{ + rpc_allow_subnets.clear(); + rpc_allow_subnets.push_back(CSubNet("127.0.0.0/8")); // always allow IPv4 local subnet + rpc_allow_subnets.push_back(CSubNet("::1")); // always allow IPv6 localhost + if (mapMultiArgs.count("-rpcallowip")) { + const std::vector<std::string>& vAllow = mapMultiArgs["-rpcallowip"]; + BOOST_FOREACH (std::string strAllow, vAllow) { + CSubNet subnet(strAllow); + if (!subnet.IsValid()) { + uiInterface.ThreadSafeMessageBox( + strprintf("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24).", strAllow), + "", CClientUIInterface::MSG_ERROR); + return false; + } + rpc_allow_subnets.push_back(subnet); + } + } + std::string strAllowed; + BOOST_FOREACH (const CSubNet& subnet, rpc_allow_subnets) + strAllowed += subnet.ToString() + " "; + LogPrint("http", "Allowing HTTP connections from: %s\n", strAllowed); + return true; +} + +/** HTTP request method as string - use for logging only */ +static std::string RequestMethodString(HTTPRequest::RequestMethod m) +{ + switch (m) { + case HTTPRequest::GET: + return "GET"; + break; + case HTTPRequest::POST: + return "POST"; + break; + case HTTPRequest::HEAD: + return "HEAD"; + break; + case HTTPRequest::PUT: + return "PUT"; + break; + default: + return "unknown"; + } +} + +/** HTTP request callback */ +static void http_request_cb(struct evhttp_request* req, void* arg) +{ + std::auto_ptr<HTTPRequest> hreq(new HTTPRequest(req)); + + LogPrint("http", "Received a %s request for %s from %s\n", + RequestMethodString(hreq->GetRequestMethod()), hreq->GetURI(), hreq->GetPeer().ToString()); + + // Early address-based allow check + if (!ClientAllowed(hreq->GetPeer())) { + hreq->WriteReply(HTTP_FORBIDDEN); + return; + } + + // Early reject unknown HTTP methods + if (hreq->GetRequestMethod() == HTTPRequest::UNKNOWN) { + hreq->WriteReply(HTTP_BADMETHOD); + return; + } + + // Find registered handler for prefix + std::string strURI = hreq->GetURI(); + std::string path; + std::vector<HTTPPathHandler>::const_iterator i = pathHandlers.begin(); + std::vector<HTTPPathHandler>::const_iterator iend = pathHandlers.end(); + for (; i != iend; ++i) { + bool match = false; + if (i->exactMatch) + match = (strURI == i->prefix); + else + match = (strURI.substr(0, i->prefix.size()) == i->prefix); + if (match) { + path = strURI.substr(i->prefix.size()); + break; + } + } + + // Dispatch to worker thread + if (i != iend) { + std::auto_ptr<HTTPWorkItem> item(new HTTPWorkItem(hreq.release(), path, i->handler)); + assert(workQueue); + if (workQueue->Enqueue(item.get())) + item.release(); /* if true, queue took ownership */ + else + item->req->WriteReply(HTTP_INTERNAL, "Work queue depth exceeded"); + } else { + hreq->WriteReply(HTTP_NOTFOUND); + } +} + +/** Event dispatcher thread */ +static void ThreadHTTP(struct event_base* base, struct evhttp* http) +{ + RenameThread("bitcoin-http"); + LogPrint("http", "Entering http event loop\n"); + event_base_dispatch(base); + // Event loop will be interrupted by InterruptHTTPServer() + LogPrint("http", "Exited http event loop\n"); +} + +/** Bind HTTP server to specified addresses */ +static bool HTTPBindAddresses(struct evhttp* http) +{ + int defaultPort = GetArg("-rpcport", BaseParams().RPCPort()); + int nBound = 0; + std::vector<std::pair<std::string, uint16_t> > endpoints; + + // Determine what addresses to bind to + if (!mapArgs.count("-rpcallowip")) { // Default to loopback if not allowing external IPs + endpoints.push_back(std::make_pair("::1", defaultPort)); + endpoints.push_back(std::make_pair("127.0.0.1", defaultPort)); + if (mapArgs.count("-rpcbind")) { + LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n"); + } + } else if (mapArgs.count("-rpcbind")) { // Specific bind address + const std::vector<std::string>& vbind = mapMultiArgs["-rpcbind"]; + for (std::vector<std::string>::const_iterator i = vbind.begin(); i != vbind.end(); ++i) { + int port = defaultPort; + std::string host; + SplitHostPort(*i, port, host); + endpoints.push_back(std::make_pair(host, port)); + } + } else { // No specific bind address specified, bind to any + endpoints.push_back(std::make_pair("::", defaultPort)); + endpoints.push_back(std::make_pair("0.0.0.0", defaultPort)); + } + + // Bind addresses + for (std::vector<std::pair<std::string, uint16_t> >::iterator i = endpoints.begin(); i != endpoints.end(); ++i) { + LogPrint("http", "Binding RPC on address %s port %i\n", i->first, i->second); + if (evhttp_bind_socket(http, i->first.empty() ? NULL : i->first.c_str(), i->second) == 0) { + nBound += 1; + } else { + LogPrintf("Binding RPC on address %s port %i failed.\n", i->first, i->second); + } + } + return nBound > 0; +} + +/** Simple wrapper to set thread name and run work queue */ +static void HTTPWorkQueueRun(WorkQueue<HTTPClosure>* queue) +{ + RenameThread("bitcoin-httpworker"); + queue->Run(); +} + +/** libevent event log callback */ +static void libevent_log_cb(int severity, const char *msg) +{ + if (severity >= EVENT_LOG_WARN) // Log warn messages and higher without debug category + LogPrintf("libevent: %s\n", msg); + else + LogPrint("libevent", "libevent: %s\n", msg); +} + +bool InitHTTPServer() +{ + struct evhttp* http = 0; + struct event_base* base = 0; + + if (!InitHTTPAllowList()) + return false; + + if (GetBoolArg("-rpcssl", false)) { + uiInterface.ThreadSafeMessageBox( + "SSL mode for RPC (-rpcssl) is no longer supported.", + "", CClientUIInterface::MSG_ERROR); + return false; + } + + // Redirect libevent's logging to our own log + event_set_log_callback(&libevent_log_cb); +#if LIBEVENT_VERSION_NUMBER >= 0x02010100 + // If -debug=libevent, set full libevent debugging. + // Otherwise, disable all libevent debugging. + if (LogAcceptCategory("libevent")) + event_enable_debug_logging(EVENT_DBG_ALL); + else + event_enable_debug_logging(EVENT_DBG_NONE); +#endif +#ifdef WIN32 + evthread_use_windows_threads(); +#else + evthread_use_pthreads(); +#endif + + base = event_base_new(); // XXX RAII + if (!base) { + LogPrintf("Couldn't create an event_base: exiting\n"); + return false; + } + + /* Create a new evhttp object to handle requests. */ + http = evhttp_new(base); // XXX RAII + if (!http) { + LogPrintf("couldn't create evhttp. Exiting.\n"); + event_base_free(base); + return false; + } + + evhttp_set_timeout(http, GetArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); + evhttp_set_max_body_size(http, MAX_SIZE); + evhttp_set_gencb(http, http_request_cb, NULL); + + if (!HTTPBindAddresses(http)) { + LogPrintf("Unable to bind any endpoint for RPC server\n"); + evhttp_free(http); + event_base_free(base); + return false; + } + + LogPrint("http", "Initialized HTTP server\n"); + int workQueueDepth = std::max((long)GetArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); + LogPrintf("HTTP: creating work queue of depth %d\n", workQueueDepth); + + workQueue = new WorkQueue<HTTPClosure>(workQueueDepth); + eventBase = base; + eventHTTP = http; + return true; +} + +bool StartHTTPServer(boost::thread_group& threadGroup) +{ + LogPrint("http", "Starting HTTP server\n"); + int rpcThreads = std::max((long)GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); + LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); + threadGroup.create_thread(boost::bind(&ThreadHTTP, eventBase, eventHTTP)); + + for (int i = 0; i < rpcThreads; i++) + threadGroup.create_thread(boost::bind(&HTTPWorkQueueRun, workQueue)); + return true; +} + +void InterruptHTTPServer() +{ + LogPrint("http", "Interrupting HTTP server\n"); + if (eventBase) + event_base_loopbreak(eventBase); + if (workQueue) + workQueue->Interrupt(); +} + +void StopHTTPServer() +{ + LogPrint("http", "Stopping HTTP server\n"); + delete workQueue; + if (eventHTTP) { + evhttp_free(eventHTTP); + eventHTTP = 0; + } + if (eventBase) { + event_base_free(eventBase); + eventBase = 0; + } +} + +struct event_base* EventBase() +{ + return eventBase; +} + +static void httpevent_callback_fn(evutil_socket_t, short, void* data) +{ + // Static handler: simply call inner handler + HTTPEvent *self = ((HTTPEvent*)data); + self->handler(); + if (self->deleteWhenTriggered) + delete self; +} + +HTTPEvent::HTTPEvent(struct event_base* base, bool deleteWhenTriggered, const boost::function<void(void)>& handler): + deleteWhenTriggered(deleteWhenTriggered), handler(handler) +{ + ev = event_new(base, -1, 0, httpevent_callback_fn, this); + assert(ev); +} +HTTPEvent::~HTTPEvent() +{ + event_free(ev); +} +void HTTPEvent::trigger(struct timeval* tv) +{ + if (tv == NULL) + event_active(ev, 0, 0); // immediately trigger event in main thread + else + evtimer_add(ev, tv); // trigger after timeval passed +} +HTTPRequest::HTTPRequest(struct evhttp_request* req) : req(req), + replySent(false) +{ +} +HTTPRequest::~HTTPRequest() +{ + if (!replySent) { + // Keep track of whether reply was sent to avoid request leaks + LogPrintf("%s: Unhandled request\n", __func__); + WriteReply(HTTP_INTERNAL, "Unhandled request"); + } + // evhttpd cleans up the request, as long as a reply was sent. +} + +std::pair<bool, std::string> HTTPRequest::GetHeader(const std::string& hdr) +{ + const struct evkeyvalq* headers = evhttp_request_get_input_headers(req); + assert(headers); + const char* val = evhttp_find_header(headers, hdr.c_str()); + if (val) + return std::make_pair(true, val); + else + return std::make_pair(false, ""); +} + +std::string HTTPRequest::ReadBody() +{ + struct evbuffer* buf = evhttp_request_get_input_buffer(req); + if (!buf) + return ""; + size_t size = evbuffer_get_length(buf); + /** Trivial implementation: if this is ever a performance bottleneck, + * internal copying can be avoided in multi-segment buffers by using + * evbuffer_peek and an awkward loop. Though in that case, it'd be even + * better to not copy into an intermediate string but use a stream + * abstraction to consume the evbuffer on the fly in the parsing algorithm. + */ + const char* data = (const char*)evbuffer_pullup(buf, size); + if (!data) // returns NULL in case of empty buffer + return ""; + std::string rv(data, size); + evbuffer_drain(buf, size); + return rv; +} + +void HTTPRequest::WriteHeader(const std::string& hdr, const std::string& value) +{ + struct evkeyvalq* headers = evhttp_request_get_output_headers(req); + assert(headers); + evhttp_add_header(headers, hdr.c_str(), value.c_str()); +} + +/** Closure sent to main thread to request a reply to be sent to + * a HTTP request. + * Replies must be sent in the main loop in the main http thread, + * this cannot be done from worker threads. + */ +void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) +{ + assert(!replySent && req); + // Send event to main http thread to send reply message + struct evbuffer* evb = evhttp_request_get_output_buffer(req); + assert(evb); + evbuffer_add(evb, strReply.data(), strReply.size()); + HTTPEvent* ev = new HTTPEvent(eventBase, true, + boost::bind(evhttp_send_reply, req, nStatus, (const char*)NULL, (struct evbuffer *)NULL)); + ev->trigger(0); + replySent = true; + req = 0; // transferred back to main thread +} + +CService HTTPRequest::GetPeer() +{ + evhttp_connection* con = evhttp_request_get_connection(req); + CService peer; + if (con) { + // evhttp retains ownership over returned address string + const char* address = ""; + uint16_t port = 0; + evhttp_connection_get_peer(con, (char**)&address, &port); + peer = CService(address, port); + } + return peer; +} + +std::string HTTPRequest::GetURI() +{ + return evhttp_request_get_uri(req); +} + +HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod() +{ + switch (evhttp_request_get_command(req)) { + case EVHTTP_REQ_GET: + return GET; + break; + case EVHTTP_REQ_POST: + return POST; + break; + case EVHTTP_REQ_HEAD: + return HEAD; + break; + case EVHTTP_REQ_PUT: + return PUT; + break; + default: + return UNKNOWN; + break; + } +} + +void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler) +{ + LogPrint("http", "Registering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); + pathHandlers.push_back(HTTPPathHandler(prefix, exactMatch, handler)); +} + +void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch) +{ + std::vector<HTTPPathHandler>::iterator i = pathHandlers.begin(); + std::vector<HTTPPathHandler>::iterator iend = pathHandlers.end(); + for (; i != iend; ++i) + if (i->prefix == prefix && i->exactMatch == exactMatch) + break; + if (i != iend) + { + LogPrint("http", "Unregistering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); + pathHandlers.erase(i); + } +} + diff --git a/src/httpserver.h b/src/httpserver.h new file mode 100644 index 0000000000..b377dc19fc --- /dev/null +++ b/src/httpserver.h @@ -0,0 +1,149 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_HTTPSERVER_H +#define BITCOIN_HTTPSERVER_H + +#include <string> +#include <stdint.h> +#include <boost/thread.hpp> +#include <boost/scoped_ptr.hpp> +#include <boost/function.hpp> + +static const int DEFAULT_HTTP_THREADS=4; +static const int DEFAULT_HTTP_WORKQUEUE=16; +static const int DEFAULT_HTTP_SERVER_TIMEOUT=30; + +struct evhttp_request; +struct event_base; +class CService; +class HTTPRequest; + +/** Initialize HTTP server. + * Call this before RegisterHTTPHandler or EventBase(). + */ +bool InitHTTPServer(); +/** Start HTTP server. + * This is separate from InitHTTPServer to give users race-condition-free time + * to register their handlers between InitHTTPServer and StartHTTPServer. + */ +bool StartHTTPServer(boost::thread_group& threadGroup); +/** Interrupt HTTP server threads */ +void InterruptHTTPServer(); +/** Stop HTTP server */ +void StopHTTPServer(); + +/** Handler for requests to a certain HTTP path */ +typedef boost::function<void(HTTPRequest* req, const std::string &)> HTTPRequestHandler; +/** Register handler for prefix. + * If multiple handlers match a prefix, the first-registered one will + * be invoked. + */ +void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler); +/** Unregister handler for prefix */ +void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch); + +/** Return evhttp event base. This can be used by submodules to + * queue timers or custom events. + */ +struct event_base* EventBase(); + +/** In-flight HTTP request. + * Thin C++ wrapper around evhttp_request. + */ +class HTTPRequest +{ +private: + struct evhttp_request* req; + bool replySent; + +public: + HTTPRequest(struct evhttp_request* req); + ~HTTPRequest(); + + enum RequestMethod { + UNKNOWN, + GET, + POST, + HEAD, + PUT + }; + + /** Get requested URI. + */ + std::string GetURI(); + + /** Get CService (address:ip) for the origin of the http request. + */ + CService GetPeer(); + + /** Get request method. + */ + RequestMethod GetRequestMethod(); + + /** + * Get the request header specified by hdr, or an empty string. + * Return an pair (isPresent,string). + */ + std::pair<bool, std::string> GetHeader(const std::string& hdr); + + /** + * Read request body. + * + * @note As this consumes the underlying buffer, call this only once. + * Repeated calls will return an empty string. + */ + std::string ReadBody(); + + /** + * Write output header. + * + * @note call this before calling WriteErrorReply or Reply. + */ + void WriteHeader(const std::string& hdr, const std::string& value); + + /** + * Write HTTP reply. + * nStatus is the HTTP status code to send. + * strReply is the body of the reply. Keep it empty to send a standard message. + * + * @note Can be called only once. As this will give the request back to the + * main thread, do not call any other HTTPRequest methods after calling this. + */ + void WriteReply(int nStatus, const std::string& strReply = ""); +}; + +/** Event handler closure. + */ +class HTTPClosure +{ +public: + virtual void operator()() = 0; + virtual ~HTTPClosure() {} +}; + +/** Event class. This can be used either as an cross-thread trigger or as a timer. + */ +class HTTPEvent +{ +public: + /** Create a new event. + * deleteWhenTriggered deletes this event object after the event is triggered (and the handler called) + * handler is the handler to call when the event is triggered. + */ + HTTPEvent(struct event_base* base, bool deleteWhenTriggered, const boost::function<void(void)>& handler); + ~HTTPEvent(); + + /** Trigger the event. If tv is 0, trigger it immediately. Otherwise trigger it after + * the given time has elapsed. + */ + void trigger(struct timeval* tv); + + bool deleteWhenTriggered; + boost::function<void(void)> handler; +private: + struct event* ev; +}; + +#endif // BITCOIN_HTTPSERVER_H diff --git a/src/init.cpp b/src/init.cpp index 085e04fdfd..7c3c18acd2 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -16,6 +16,8 @@ #include "checkpoints.h" #include "compat/sanity.h" #include "consensus/validation.h" +#include "httpserver.h" +#include "httprpc.h" #include "key.h" #include "main.h" #include "miner.h" @@ -36,7 +38,6 @@ #include "wallet/wallet.h" #include "wallet/walletdb.h" #endif - #include <stdint.h> #include <stdio.h> @@ -53,6 +54,10 @@ #include <boost/thread.hpp> #include <openssl/crypto.h> +#if ENABLE_ZMQ +#include "zmq/zmqnotificationinterface.h" +#endif + using namespace std; #ifdef ENABLE_WALLET @@ -60,6 +65,10 @@ CWallet* pwalletMain = NULL; #endif bool fFeeEstimatesInitialized = false; +#if ENABLE_ZMQ +static CZMQNotificationInterface* pzmqNotificationInterface = NULL; +#endif + #ifdef WIN32 // Win32 LevelDB doesn't use filedescriptors, and the ones used for // accessing block files don't count towards the fd_set size limit @@ -144,6 +153,15 @@ public: static CCoinsViewDB *pcoinsdbview = NULL; static CCoinsViewErrorCatcher *pcoinscatcher = NULL; +void Interrupt(boost::thread_group& threadGroup) +{ + InterruptHTTPServer(); + InterruptHTTPRPC(); + InterruptRPC(); + InterruptREST(); + threadGroup.interrupt_all(); +} + void Shutdown() { LogPrintf("%s: In progress...\n", __func__); @@ -158,7 +176,11 @@ void Shutdown() /// module was initialized. RenameThread("bitcoin-shutoff"); mempool.AddTransactionsUpdated(1); - StopRPCThreads(); + + StopHTTPRPC(); + StopREST(); + StopRPC(); + StopHTTPServer(); #ifdef ENABLE_WALLET if (pwalletMain) pwalletMain->Flush(false); @@ -196,6 +218,16 @@ void Shutdown() if (pwalletMain) pwalletMain->Flush(true); #endif + +#if ENABLE_ZMQ + if (pzmqNotificationInterface) { + UnregisterValidationInterface(pzmqNotificationInterface); + pzmqNotificationInterface->Shutdown(); + delete pzmqNotificationInterface; + pzmqNotificationInterface = NULL; + } +#endif + #ifndef WIN32 try { boost::filesystem::remove(GetPidFile()); @@ -293,7 +325,7 @@ std::string HelpMessage(HelpMessageMode mode) #ifndef WIN32 strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), "bitcoind.pid")); #endif - strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex. " + strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode is incompatible with -txindex and -rescan. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files on startup")); @@ -335,7 +367,6 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-whitebind=<addr>", _("Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6")); strUsage += HelpMessageOpt("-whitelist=<netmask>", _("Whitelist peers connecting from the given netmask or IP address. Can be specified multiple times.") + " " + _("Whitelisted peers cannot be DoS banned and their transactions are always relayed, even if they are already in the mempool, useful e.g. for a gateway")); - strUsage += HelpMessageOpt("-whiteconnections=<n>", strprintf(_("Reserve this many inbound connections for whitelisted peers (default: %d)"), 0)); #ifdef ENABLE_WALLET strUsage += HelpMessageGroup(_("Wallet options:")); @@ -361,6 +392,14 @@ std::string HelpMessage(HelpMessageMode mode) " " + _("(1 = keep tx meta data e.g. account owner and payment request information, 2 = drop tx meta data)")); #endif +#if ENABLE_ZMQ + strUsage += HelpMessageGroup(_("ZeroMQ notification options:")); + strUsage += HelpMessageOpt("-zmqpubhashblock=<address>", _("Enable publish hash block in <address>")); + strUsage += HelpMessageOpt("-zmqpubhashtransaction=<address>", _("Enable publish hash transaction in <address>")); + strUsage += HelpMessageOpt("-zmqpubrawblock=<address>", _("Enable publish raw block in <address>")); + strUsage += HelpMessageOpt("-zmqpubrawtransaction=<address>", _("Enable publish raw transaction in <address>")); +#endif + strUsage += HelpMessageGroup(_("Debugging/Testing options:")); if (showDebug) { @@ -372,8 +411,12 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-fuzzmessagestest=<n>", "Randomly fuzz 1 of every <n> network messages"); strUsage += HelpMessageOpt("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", 1)); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", 0)); + strUsage += HelpMessageOpt("-limitancestorcount=<n>", strprintf("Do not accept transactions if number of in-mempool ancestors is <n> or more (default: %u)", DEFAULT_ANCESTOR_LIMIT)); + strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT)); + strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); + strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); } - string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, mempoolrej, net, proxy, prune"; // Don't translate these and qt below + string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, mempoolrej, net, proxy, prune, http, libevent"; // Don't translate these and qt below if (mode == HMM_BITCOIN_QT) debugCategories += ", qt"; strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " + @@ -423,14 +466,11 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-rpcpassword=<pw>", _("Password for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcport=<port>", strprintf(_("Listen for JSON-RPC connections on <port> (default: %u or testnet: %u)"), 8332, 18332)); strUsage += HelpMessageOpt("-rpcallowip=<ip>", _("Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times")); - strUsage += HelpMessageOpt("-rpcthreads=<n>", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), 4)); - strUsage += HelpMessageOpt("-rpckeepalive", strprintf(_("RPC support for HTTP persistent connections (default: %d)"), 1)); - - strUsage += HelpMessageGroup(_("RPC SSL options: (see the Bitcoin Wiki for SSL setup instructions)")); - strUsage += HelpMessageOpt("-rpcssl", _("Use OpenSSL (https) for JSON-RPC connections")); - strUsage += HelpMessageOpt("-rpcsslcertificatechainfile=<file.cert>", strprintf(_("Server certificate file (default: %s)"), "server.cert")); - strUsage += HelpMessageOpt("-rpcsslprivatekeyfile=<file.pem>", strprintf(_("Server private key (default: %s)"), "server.pem")); - strUsage += HelpMessageOpt("-rpcsslciphers=<ciphers>", strprintf(_("Acceptable ciphers (default: %s)"), "TLSv1.2+HIGH:TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!3DES:@STRENGTH")); + strUsage += HelpMessageOpt("-rpcthreads=<n>", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), DEFAULT_HTTP_THREADS)); + if (showDebug) { + strUsage += HelpMessageOpt("-rpcworkqueue=<n>", strprintf("Set the depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE)); + strUsage += HelpMessageOpt("-rpcservertimeout=<n>", strprintf("Timeout during HTTP requests (default: %d)", DEFAULT_HTTP_SERVER_TIMEOUT)); + } if (mode == HMM_BITCOIN_QT) { @@ -602,6 +642,23 @@ bool InitSanityCheck(void) return true; } +bool AppInitServers(boost::thread_group& threadGroup) +{ + RPCServer::OnStopped(&OnRPCStopped); + RPCServer::OnPreCommand(&OnRPCPreCommand); + if (!InitHTTPServer()) + return false; + if (!StartRPC()) + return false; + if (!StartHTTPRPC()) + return false; + if (GetBoolArg("-rest", false) && !StartREST()) + return false; + if (!StartHTTPServer(threadGroup)) + return false; + return true; +} + /** Initialize bitcoin. * @pre Parameters should be parsed and config file should be read. */ @@ -629,17 +686,12 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) typedef BOOL (WINAPI *PSETPROCDEPPOL)(DWORD); PSETPROCDEPPOL setProcDEPPol = (PSETPROCDEPPOL)GetProcAddress(GetModuleHandleA("Kernel32.dll"), "SetProcessDEPPolicy"); if (setProcDEPPol != NULL) setProcDEPPol(PROCESS_DEP_ENABLE); - - // Initialize Windows Sockets - WSADATA wsadata; - int ret = WSAStartup(MAKEWORD(2,2), &wsadata); - if (ret != NO_ERROR || LOBYTE(wsadata.wVersion ) != 2 || HIBYTE(wsadata.wVersion) != 2) - { - return InitError(strprintf("Error: Winsock library failed to start (WSAStartup returned error %d)", ret)); - } #endif -#ifndef WIN32 + if (!SetupNetworking()) + return InitError("Error: Initializing networking failed"); + +#ifndef WIN32 if (GetBoolArg("-sysperms", false)) { #ifdef ENABLE_WALLET if (!GetBoolArg("-disablewallet", false)) @@ -664,11 +716,9 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) sa_hup.sa_flags = 0; sigaction(SIGHUP, &sa_hup, NULL); -#if defined (__SVR4) && defined (__sun) - // ignore SIGPIPE on Solaris + // Ignore SIGPIPE, otherwise it will bring the daemon down if the client closes unexpectedly signal(SIGPIPE, SIG_IGN); #endif -#endif // ********************************************************* Step 2: parameter interactions const CChainParams& chainparams = Params(); @@ -754,25 +804,6 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) int nBind = std::max((int)mapArgs.count("-bind") + (int)mapArgs.count("-whitebind"), 1); int nUserMaxConnections = GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); nMaxConnections = std::max(nUserMaxConnections, 0); - int nUserWhiteConnections = GetArg("-whiteconnections", 0); - nWhiteConnections = std::max(nUserWhiteConnections, 0); - - if ((mapArgs.count("-whitelist")) || (mapArgs.count("-whitebind"))) { - if (!(mapArgs.count("-maxconnections"))) { - // User is using whitelist feature, - // but did not specify -maxconnections parameter. - // Silently increase the default to compensate, - // so that the whitelist connection reservation feature - // does not inadvertently reduce the default - // inbound connection capacity of the network. - nMaxConnections += nWhiteConnections; - } - } else { - // User not using whitelist feature. - // Silently disable connection reservation, - // for the same reason as above. - nWhiteConnections = 0; - } // Trim requested connection counts, to fit into system limitations nMaxConnections = std::max(std::min(nMaxConnections, (int)(FD_SETSIZE - nBind - MIN_CORE_FILEDESCRIPTORS)), 0); @@ -784,13 +815,6 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) if (nMaxConnections < nUserMaxConnections) InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); - // Connection capacity is prioritized in this order: - // outbound connections (hardcoded to 8), - // then whitelisted connections, - // then non-whitelisted connections get whatever's left (if any). - if ((nWhiteConnections > 0) && (nWhiteConnections >= (nMaxConnections - 8))) - InitWarning(strprintf(_("All non-whitelisted incoming connections will be dropped, because -whiteconnections is %d and -maxconnections is only %d."), nWhiteConnections, nMaxConnections)); - // ********************************************************* Step 3: parameter-to-internal-flags fDebug = !mapMultiArgs["-debug"].empty(); @@ -921,6 +945,9 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) // Option to startup with mocktime set (used for regression testing): SetMockTime(GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op + if (GetBoolArg("-peerbloomfilters", true)) + nLocalServices |= NODE_BLOOM; + // ********************************************************* Step 4: application initialization: dir lock, daemonize, pidfile, debug log // Initialize elliptic curve code @@ -968,8 +995,6 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) LogPrintf("Using data directory %s\n", strDataDir); LogPrintf("Using config file %s\n", GetConfigFile().string()); LogPrintf("Using at most %i connections (%i file descriptors available)\n", nMaxConnections, nFD); - if (nWhiteConnections > 0) - LogPrintf("Reserving %i of these connections for whitelisted inbound peers\n", nWhiteConnections); std::ostringstream strErrors; LogPrintf("Using %u threads for script verification\n", nScriptCheckThreads); @@ -990,9 +1015,8 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) if (fServer) { uiInterface.InitMessage.connect(SetRPCWarmupStatus); - RPCServer::OnStopped(&OnRPCStopped); - RPCServer::OnPreCommand(&OnRPCPreCommand); - StartRPCThreads(); + if (!AppInitServers(threadGroup)) + return InitError(_("Unable to start HTTP server. See debug log for details.")); } int64_t nStart; @@ -1020,8 +1044,15 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) RegisterNodeSignals(GetNodeSignals()); - // format user agent, check total size - strSubVersion = FormatSubVersion(CLIENT_NAME, CLIENT_VERSION, mapMultiArgs.count("-uacomment") ? mapMultiArgs["-uacomment"] : std::vector<string>()); + // sanitize comments per BIP-0014, format user agent and check total size + std::vector<string> uacomments; + BOOST_FOREACH(string cmt, mapMultiArgs["-uacomment"]) + { + if (cmt != SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)) + return InitError(strprintf("User Agent comment (%s) contains unsafe characters.", cmt)); + uacomments.push_back(SanitizeString(cmt, SAFE_CHARS_UA_COMMENT)); + } + strSubVersion = FormatSubVersion(CLIENT_NAME, CLIENT_VERSION, uacomments); if (strSubVersion.size() > MAX_SUBVERSION_LENGTH) { return InitError(strprintf("Total length of network version string %i exceeds maximum of %i characters. Reduce the number and/or size of uacomments.", strSubVersion.size(), MAX_SUBVERSION_LENGTH)); @@ -1128,6 +1159,15 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) BOOST_FOREACH(const std::string& strDest, mapMultiArgs["-seednode"]) AddOneShot(strDest); +#if ENABLE_ZMQ + pzmqNotificationInterface = CZMQNotificationInterface::CreateWithArguments(mapArgs); + + if (pzmqNotificationInterface) { + pzmqNotificationInterface->Initialize(); + RegisterValidationInterface(pzmqNotificationInterface); + } +#endif + // ********************************************************* Step 7: load block chain fReindex = GetBoolArg("-reindex", false); diff --git a/src/init.h b/src/init.h index dcb2b29360..8cd51b0286 100644 --- a/src/init.h +++ b/src/init.h @@ -20,6 +20,8 @@ extern CWallet* pwalletMain; void StartShutdown(); bool ShutdownRequested(); +/** Interrupt threads */ +void Interrupt(boost::thread_group& threadGroup); void Shutdown(); bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler); diff --git a/src/main.cpp b/src/main.cpp index 33b57a5285..1df4c9fa36 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -921,6 +921,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa REJECT_HIGHFEE, "absurdly-high-fee", strprintf("%d > %d", nFees, ::minRelayTxFee.GetFee(nSize) * 10000)); + // Calculate in-mempool ancestors, up to a limit. + CTxMemPool::setEntries setAncestors; + size_t nLimitAncestors = GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); + size_t nLimitAncestorSize = GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000; + size_t nLimitDescendants = GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); + size_t nLimitDescendantSize = GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000; + std::string errString; + if (!pool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) { + return state.DoS(0, false, REJECT_NONSTANDARD, "too-long-mempool-chain", false, errString); + } + // Check against previous transactions // This is done last to help prevent CPU exhaustion denial-of-service attacks. if (!CheckInputs(tx, state, view, true, STANDARD_SCRIPT_VERIFY_FLAGS, true)) @@ -942,7 +953,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa } // Store transaction in memory - pool.addUnchecked(hash, entry, !IsInitialBlockDownload()); + pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload()); } SyncWithWallets(tx, NULL); @@ -954,47 +965,45 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock, bool fAllowSlow) { CBlockIndex *pindexSlow = NULL; + + LOCK(cs_main); + + if (mempool.lookup(hash, txOut)) { - LOCK(cs_main); - { - if (mempool.lookup(hash, txOut)) - { - return true; - } - } + return true; + } - if (fTxIndex) { - CDiskTxPos postx; - if (pblocktree->ReadTxIndex(hash, postx)) { - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); - if (file.IsNull()) - return error("%s: OpenBlockFile failed", __func__); - CBlockHeader header; - try { - file >> header; - fseek(file.Get(), postx.nTxOffset, SEEK_CUR); - file >> txOut; - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); - } - hashBlock = header.GetHash(); - if (txOut.GetHash() != hash) - return error("%s: txid mismatch", __func__); - return true; + if (fTxIndex) { + CDiskTxPos postx; + if (pblocktree->ReadTxIndex(hash, postx)) { + CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) + return error("%s: OpenBlockFile failed", __func__); + CBlockHeader header; + try { + file >> header; + fseek(file.Get(), postx.nTxOffset, SEEK_CUR); + file >> txOut; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } + hashBlock = header.GetHash(); + if (txOut.GetHash() != hash) + return error("%s: txid mismatch", __func__); + return true; } + } - if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - int nHeight = -1; - { - CCoinsViewCache &view = *pcoinsTip; - const CCoins* coins = view.AccessCoins(hash); - if (coins) - nHeight = coins->nHeight; - } - if (nHeight > 0) - pindexSlow = chainActive[nHeight]; + if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it + int nHeight = -1; + { + CCoinsViewCache &view = *pcoinsTip; + const CCoins* coins = view.AccessCoins(hash); + if (coins) + nHeight = coins->nHeight; } + if (nHeight > 0) + pindexSlow = chainActive[nHeight]; } if (pindexSlow) { @@ -2033,13 +2042,23 @@ bool static DisconnectTip(CValidationState &state) { if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; // Resurrect mempool transactions from the disconnected block. + std::vector<uint256> vHashUpdate; BOOST_FOREACH(const CTransaction &tx, block.vtx) { // ignore validation errors in resurrected transactions list<CTransaction> removed; CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL)) + if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL)) { mempool.remove(tx, removed, true); + } else if (mempool.exists(tx.GetHash())) { + vHashUpdate.push_back(tx.GetHash()); + } } + // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // no in-mempool children, which is generally not true when adding + // previously-confirmed transactions back to the mempool. + // UpdateTransactionsFromBlock finds descendants of any transactions in this + // block that were added back and cleans up the mempool state. + mempool.UpdateTransactionsFromBlock(vHashUpdate); mempool.removeCoinbaseSpends(pcoinsTip, pindexDelete->nHeight); mempool.check(pcoinsTip); // Update chainActive and related variables. @@ -2294,15 +2313,14 @@ bool ActivateBestChain(CValidationState &state, const CBlock *pblock) { int nBlockEstimate = 0; if (fCheckpointsEnabled) nBlockEstimate = Checkpoints::GetTotalBlocksEstimate(chainParams.Checkpoints()); - // Don't relay blocks if pruning -- could cause a peer to try to download, resulting - // in a stalled download if the block file is pruned before the request. - if (nLocalServices & NODE_NETWORK) { + { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) pnode->PushInventory(CInv(MSG_BLOCK, hashNewTip)); } // Notify external listeners about the new tip. + GetMainSignals().UpdatedBlockTip(hashNewTip); uiInterface.NotifyBlockTip(hashNewTip); } } while(pindexMostWork != chainActive.Tip()); @@ -2565,6 +2583,9 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo { // These are checks that are independent of context. + if (block.fChecked) + return true; + // Check that the header is valid (particularly PoW). This is mostly // redundant with the call in AcceptBlockHeader. if (!CheckBlockHeader(block, state, fCheckPOW)) @@ -2573,7 +2594,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo // Check the merkle root. if (fCheckMerkleRoot) { bool mutated; - uint256 hashMerkleRoot2 = block.BuildMerkleTree(&mutated); + uint256 hashMerkleRoot2 = block.ComputeMerkleRoot(&mutated); if (block.hashMerkleRoot != hashMerkleRoot2) return state.DoS(100, error("CheckBlock(): hashMerkleRoot mismatch"), REJECT_INVALID, "bad-txnmrklroot", true); @@ -2620,6 +2641,9 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo return state.DoS(100, error("CheckBlock(): out-of-bounds SigOpCount"), REJECT_INVALID, "bad-blk-sigops", true); + if (fCheckPOW && fCheckMerkleRoot) + block.fChecked = true; + return true; } @@ -4176,6 +4200,14 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LogPrint("net", " getblocks stopping at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } + // If pruning, don't inv blocks unless we have on disk and are likely to still have + // for some reasonable time window (1 hour) that block relay might require. + const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / chainparams.GetConsensus().nPowTargetSpacing; + if (fPruneMode && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->nHeight <= chainActive.Tip()->nHeight - nPrunedBlocksLikelyToHave)) + { + LogPrint("net", " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + break; + } pfrom->PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); if (--nLimit <= 0) { @@ -4254,10 +4286,10 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, RelayTransaction(tx); vWorkQueue.push_back(inv.hash); - LogPrint("mempool", "AcceptToMemoryPool: peer=%d %s: accepted %s (poolsz %u)\n", - pfrom->id, pfrom->cleanSubVer, + LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u)\n", + pfrom->id, tx.GetHash().ToString(), - mempool.mapTx.size()); + mempool.size()); // Recursively process any orphan transactions that depended on this one set<NodeId> setMisbehaving; @@ -4344,8 +4376,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int nDoS = 0; if (state.IsInvalid(nDoS)) { - LogPrint("mempoolrej", "%s from peer=%d %s was not accepted into the memory pool: %s\n", tx.GetHash().ToString(), - pfrom->id, pfrom->cleanSubVer, + LogPrint("mempoolrej", "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), + pfrom->id, FormatStateMessage(state)); if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P pfrom->PushMessage("reject", strCommand, state.GetRejectCode(), @@ -4522,6 +4554,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (pingUsecTime > 0) { // Successful ping time measurement, replace previous pfrom->nPingUsecTime = pingUsecTime; + pfrom->nMinPingUsecTime = std::min(pfrom->nMinPingUsecTime, pingUsecTime); } else { // This should never happen sProblem = "Timing mishap"; @@ -4545,9 +4578,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } if (!(sProblem.empty())) { - LogPrint("net", "pong peer=%d %s: %s, %x expected, %x received, %u bytes\n", + LogPrint("net", "pong peer=%d: %s, %x expected, %x received, %u bytes\n", pfrom->id, - pfrom->cleanSubVer, sProblem, pfrom->nPingNonceSent, nonce, @@ -4590,6 +4622,21 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } + else if (!(nLocalServices & NODE_BLOOM) && + (strCommand == "filterload" || + strCommand == "filteradd" || + strCommand == "filterclear") && + //TODO: Remove this line after reasonable network upgrade + pfrom->nVersion >= NO_BLOOM_VERSION) + { + if (pfrom->nVersion >= NO_BLOOM_VERSION) + Misbehaving(pfrom->GetId(), 100); + //TODO: Enable this after reasonable network upgrade + //else + // pfrom->fDisconnect = true; + } + + else if (strCommand == "filterload") { CBloomFilter filter; diff --git a/src/main.h b/src/main.h index e3479b4b3b..a6001eed8f 100644 --- a/src/main.h +++ b/src/main.h @@ -43,6 +43,14 @@ struct CNodeStateStats; static const bool DEFAULT_ALERTS = true; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; +/** Default for -limitancestorcount, max number of in-mempool ancestors */ +static const unsigned int DEFAULT_ANCESTOR_LIMIT = 100; +/** Default for -limitancestorsize, maximum kilobytes of tx + all in-mempool ancestors */ +static const unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT = 900; +/** Default for -limitdescendantcount, max number of in-mempool descendants */ +static const unsigned int DEFAULT_DESCENDANT_LIMIT = 1000; +/** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */ +static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 2500; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ diff --git a/src/memusage.h b/src/memusage.h index be3964df1b..b475c3313b 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -74,18 +74,30 @@ static inline size_t DynamicUsage(const std::vector<X>& v) return MallocUsage(v.capacity() * sizeof(X)); } -template<typename X> -static inline size_t DynamicUsage(const std::set<X>& s) +template<typename X, typename Y> +static inline size_t DynamicUsage(const std::set<X, Y>& s) { return MallocUsage(sizeof(stl_tree_node<X>)) * s.size(); } template<typename X, typename Y> -static inline size_t DynamicUsage(const std::map<X, Y>& m) +static inline size_t IncrementalDynamicUsage(const std::set<X, Y>& s) +{ + return MallocUsage(sizeof(stl_tree_node<X>)); +} + +template<typename X, typename Y, typename Z> +static inline size_t DynamicUsage(const std::map<X, Y, Z>& m) { return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)) * m.size(); } +template<typename X, typename Y, typename Z> +static inline size_t IncrementalDynamicUsage(const std::map<X, Y, Z>& m) +{ + return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)); +} + // Boost data structures template<typename X> diff --git a/src/miner.cpp b/src/miner.cpp index 9dd1d459b5..42c8bb970b 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -158,10 +158,10 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) // This vector will be sorted into a priority queue: vector<TxPriority> vecPriority; vecPriority.reserve(mempool.mapTx.size()); - for (map<uint256, CTxMemPoolEntry>::iterator mi = mempool.mapTx.begin(); + for (CTxMemPool::indexed_transaction_set::iterator mi = mempool.mapTx.begin(); mi != mempool.mapTx.end(); ++mi) { - const CTransaction& tx = mi->second.GetTx(); + const CTransaction& tx = mi->GetTx(); if (tx.IsCoinBase() || !IsFinalTx(tx, nHeight, pblock->nTime)) continue; @@ -196,7 +196,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) } mapDependers[txin.prevout.hash].push_back(porphan); porphan->setDependsOn.insert(txin.prevout.hash); - nTotalIn += mempool.mapTx[txin.prevout.hash].GetTx().vout[txin.prevout.n].nValue; + nTotalIn += mempool.mapTx.find(txin.prevout.hash)->GetTx().vout[txin.prevout.n].nValue; continue; } const CCoins* coins = view.AccessCoins(txin.prevout.hash); @@ -226,7 +226,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) porphan->feeRate = feeRate; } else - vecPriority.push_back(TxPriority(dPriority, feeRate, &mi->second.GetTx())); + vecPriority.push_back(TxPriority(dPriority, feeRate, &(mi->GetTx()))); } // Collect transactions into block @@ -368,7 +368,7 @@ void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned assert(txCoinbase.vin[0].scriptSig.size() <= 100); pblock->vtx[0] = txCoinbase; - pblock->hashMerkleRoot = pblock->BuildMerkleTree(); + pblock->hashMerkleRoot = pblock->ComputeMerkleRoot(); } ////////////////////////////////////////////////////////////////////////////// diff --git a/src/net.cpp b/src/net.cpp index 8fda5140ca..87c4f0af0a 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -81,7 +81,6 @@ uint64_t nLocalHostNonce = 0; static std::vector<ListenSocket> vhListenSocket; CAddrMan addrman; int nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; -int nWhiteConnections = 0; bool fAddressesInitialized = false; std::string strSubVersion; @@ -629,6 +628,7 @@ void CNode::copyStats(CNodeStats &stats) // Raw ping time is in microseconds, but show it to user as whole seconds (Bitcoin users should be well used to small numbers with many decimal places by now :) stats.dPingTime = (((double)nPingUsecTime) / 1e6); + stats.dPingMin = (((double)nMinPingUsecTime) / 1e6); stats.dPingWait = (((double)nPingUsecWait) / 1e6); // Leave string empty if addrLocal invalid (not filled in yet) @@ -659,7 +659,7 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes) return false; if (msg.in_data && msg.hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { - LogPrint("net", "Oversized message from peer=%i, disconnecting", GetId()); + LogPrint("net", "Oversized message from peer=%i, disconnecting\n", GetId()); return false; } @@ -776,6 +776,222 @@ void SocketSendData(CNode *pnode) static list<CNode*> vNodesDisconnected; +class CNodeRef { +public: + CNodeRef(CNode *pnode) : _pnode(pnode) { + LOCK(cs_vNodes); + _pnode->AddRef(); + } + + ~CNodeRef() { + LOCK(cs_vNodes); + _pnode->Release(); + } + + CNode& operator *() const {return *_pnode;}; + CNode* operator ->() const {return _pnode;}; + + CNodeRef& operator =(const CNodeRef& other) + { + if (this != &other) { + LOCK(cs_vNodes); + + _pnode->Release(); + _pnode = other._pnode; + _pnode->AddRef(); + } + return *this; + } + + CNodeRef(const CNodeRef& other): + _pnode(other._pnode) + { + LOCK(cs_vNodes); + _pnode->AddRef(); + } +private: + CNode *_pnode; +}; + +static bool ReverseCompareNodeMinPingTime(const CNodeRef &a, const CNodeRef &b) +{ + return a->nMinPingUsecTime > b->nMinPingUsecTime; +} + +static bool ReverseCompareNodeTimeConnected(const CNodeRef &a, const CNodeRef &b) +{ + return a->nTimeConnected > b->nTimeConnected; +} + +class CompareNetGroupKeyed +{ + std::vector<unsigned char> vchSecretKey; +public: + CompareNetGroupKeyed() + { + vchSecretKey.resize(32, 0); + GetRandBytes(vchSecretKey.data(), vchSecretKey.size()); + } + + bool operator()(const CNodeRef &a, const CNodeRef &b) + { + std::vector<unsigned char> vchGroupA, vchGroupB; + CSHA256 hashA, hashB; + std::vector<unsigned char> vchA(32), vchB(32); + + vchGroupA = a->addr.GetGroup(); + vchGroupB = b->addr.GetGroup(); + + hashA.Write(begin_ptr(vchGroupA), vchGroupA.size()); + hashB.Write(begin_ptr(vchGroupB), vchGroupB.size()); + + hashA.Write(begin_ptr(vchSecretKey), vchSecretKey.size()); + hashB.Write(begin_ptr(vchSecretKey), vchSecretKey.size()); + + hashA.Finalize(begin_ptr(vchA)); + hashB.Finalize(begin_ptr(vchB)); + + return vchA < vchB; + } +}; + +static bool AttemptToEvictConnection(bool fPreferNewConnection) { + std::vector<CNodeRef> vEvictionCandidates; + { + LOCK(cs_vNodes); + + BOOST_FOREACH(CNode *node, vNodes) { + if (node->fWhitelisted) + continue; + if (!node->fInbound) + continue; + if (node->fDisconnect) + continue; + if (node->addr.IsLocal()) + continue; + vEvictionCandidates.push_back(CNodeRef(node)); + } + } + + if (vEvictionCandidates.empty()) return false; + + // Protect connections with certain characteristics + + // Deterministically select 4 peers to protect by netgroup. + // An attacker cannot predict which netgroups will be protected. + static CompareNetGroupKeyed comparerNetGroupKeyed; + std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), comparerNetGroupKeyed); + vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); + + if (vEvictionCandidates.empty()) return false; + + // Protect the 8 nodes with the best ping times. + // An attacker cannot manipulate this metric without physically moving nodes closer to the target. + std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeMinPingTime); + vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(8, static_cast<int>(vEvictionCandidates.size())), vEvictionCandidates.end()); + + if (vEvictionCandidates.empty()) return false; + + // Protect the half of the remaining nodes which have been connected the longest. + // This replicates the existing implicit behavior. + std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeTimeConnected); + vEvictionCandidates.erase(vEvictionCandidates.end() - static_cast<int>(vEvictionCandidates.size() / 2), vEvictionCandidates.end()); + + if (vEvictionCandidates.empty()) return false; + + // Identify the network group with the most connections + std::vector<unsigned char> naMostConnections; + unsigned int nMostConnections = 0; + std::map<std::vector<unsigned char>, std::vector<CNodeRef> > mapAddrCounts; + BOOST_FOREACH(const CNodeRef &node, vEvictionCandidates) { + mapAddrCounts[node->addr.GetGroup()].push_back(node); + + if (mapAddrCounts[node->addr.GetGroup()].size() > nMostConnections) { + nMostConnections = mapAddrCounts[node->addr.GetGroup()].size(); + naMostConnections = node->addr.GetGroup(); + } + } + + // Reduce to the network group with the most connections + vEvictionCandidates = mapAddrCounts[naMostConnections]; + + // Do not disconnect peers if there is only 1 connection from their network group + if (vEvictionCandidates.size() <= 1) + // unless we prefer the new connection (for whitelisted peers) + if (!fPreferNewConnection) + return false; + + // Disconnect the most recent connection from the network group with the most connections + std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), ReverseCompareNodeTimeConnected); + vEvictionCandidates[0]->fDisconnect = true; + + return true; +} + +static void AcceptConnection(const ListenSocket& hListenSocket) { + struct sockaddr_storage sockaddr; + socklen_t len = sizeof(sockaddr); + SOCKET hSocket = accept(hListenSocket.socket, (struct sockaddr*)&sockaddr, &len); + CAddress addr; + int nInbound = 0; + int nMaxInbound = nMaxConnections - MAX_OUTBOUND_CONNECTIONS; + + if (hSocket != INVALID_SOCKET) + if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) + LogPrintf("Warning: Unknown socket family\n"); + + bool whitelisted = hListenSocket.whitelisted || CNode::IsWhitelistedRange(addr); + { + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) + if (pnode->fInbound) + nInbound++; + } + + if (hSocket == INVALID_SOCKET) + { + int nErr = WSAGetLastError(); + if (nErr != WSAEWOULDBLOCK) + LogPrintf("socket error accept failed: %s\n", NetworkErrorString(nErr)); + return; + } + + if (!IsSelectableSocket(hSocket)) + { + LogPrintf("connection from %s dropped: non-selectable socket\n", addr.ToString()); + CloseSocket(hSocket); + return; + } + + if (CNode::IsBanned(addr) && !whitelisted) + { + LogPrintf("connection from %s dropped (banned)\n", addr.ToString()); + CloseSocket(hSocket); + return; + } + + if (nInbound >= nMaxInbound) + { + if (!AttemptToEvictConnection(whitelisted)) { + // No connection to evict, disconnect the new connection + LogPrint("net", "failed to find an eviction candidate - connection dropped (full)\n"); + CloseSocket(hSocket); + return; + } + } + + CNode* pnode = new CNode(hSocket, addr, "", true); + pnode->AddRef(); + pnode->fWhitelisted = whitelisted; + + LogPrint("net", "connection from %s accepted\n", addr.ToString()); + + { + LOCK(cs_vNodes); + vNodes.push_back(pnode); + } +} + void ThreadSocketHandler() { unsigned int nPrevNodeCount = 0; @@ -933,64 +1149,7 @@ void ThreadSocketHandler() { if (hListenSocket.socket != INVALID_SOCKET && FD_ISSET(hListenSocket.socket, &fdsetRecv)) { - struct sockaddr_storage sockaddr; - socklen_t len = sizeof(sockaddr); - SOCKET hSocket = accept(hListenSocket.socket, (struct sockaddr*)&sockaddr, &len); - CAddress addr; - int nInbound = 0; - int nMaxInbound = nMaxConnections - MAX_OUTBOUND_CONNECTIONS; - - if (hSocket != INVALID_SOCKET) - if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) - LogPrintf("Warning: Unknown socket family\n"); - - bool whitelisted = hListenSocket.whitelisted || CNode::IsWhitelistedRange(addr); - { - LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) - if (pnode->fInbound) - nInbound++; - } - - if (hSocket == INVALID_SOCKET) - { - int nErr = WSAGetLastError(); - if (nErr != WSAEWOULDBLOCK) - LogPrintf("socket error accept failed: %s\n", NetworkErrorString(nErr)); - } - else if (!IsSelectableSocket(hSocket)) - { - LogPrintf("connection from %s dropped: non-selectable socket\n", addr.ToString()); - CloseSocket(hSocket); - } - else if (nInbound >= nMaxInbound) - { - LogPrint("net", "connection from %s dropped (full)\n", addr.ToString()); - CloseSocket(hSocket); - } - else if (!whitelisted && (nInbound >= (nMaxInbound - nWhiteConnections))) - { - LogPrint("net", "connection from %s dropped (non-whitelisted)\n", addr.ToString()); - CloseSocket(hSocket); - } - else if (CNode::IsBanned(addr) && !whitelisted) - { - LogPrintf("connection from %s dropped (banned)\n", addr.ToString()); - CloseSocket(hSocket); - } - else - { - CNode* pnode = new CNode(hSocket, addr, "", true); - pnode->AddRef(); - pnode->fWhitelisted = whitelisted; - - LogPrint("net", "connection from %s accepted\n", addr.ToString()); - - { - LOCK(cs_vNodes); - vNodes.push_back(pnode); - } - } + AcceptConnection(hListenSocket); } } @@ -1120,10 +1279,14 @@ void ThreadMapPort() #ifndef UPNPDISCOVER_SUCCESS /* miniupnpc 1.5 */ devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0); -#else +#elif MINIUPNPC_API_VERSION < 14 /* miniupnpc 1.6 */ int error = 0; devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, &error); +#else + /* miniupnpc 1.9.20150730 */ + int error = 0; + devlist = upnpDiscover(2000, multicastif, minissdpdpath, 0, 0, 2, &error); #endif struct UPNPUrls urls; @@ -2098,6 +2261,7 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa nPingUsecStart = 0; nPingUsecTime = 0; fPingQueued = false; + nMinPingUsecTime = std::numeric_limits<int64_t>::max(); { LOCK(cs_nLastNodeId); @@ -143,19 +143,8 @@ extern uint64_t nLocalServices; extern uint64_t nLocalHostNonce; extern CAddrMan addrman; -// The allocation of connections against the maximum allowed (nMaxConnections) -// is prioritized as follows: -// 1st: Outbound connections (MAX_OUTBOUND_CONNECTIONS) -// 2nd: Inbound connections from whitelisted peers (nWhiteConnections) -// 3rd: Inbound connections from non-whitelisted peers -// Thus, the number of connection slots for the general public to use is: -// nMaxConnections - (MAX_OUTBOUND_CONNECTIONS + nWhiteConnections) -// Any additional inbound connections beyond limits will be immediately closed - /** Maximum number of connections to simultaneously allow (aka connection slots) */ extern int nMaxConnections; -/** Number of connection slots to reserve for inbound from whitelisted peers */ -extern int nWhiteConnections; extern std::vector<CNode*> vNodes; extern CCriticalSection cs_vNodes; @@ -200,6 +189,7 @@ public: bool fWhitelisted; double dPingTime; double dPingWait; + double dPingMin; std::string addrLocal; }; @@ -395,6 +385,8 @@ public: int64_t nPingUsecStart; // Last measured round-trip time. int64_t nPingUsecTime; + // Best measured round-trip time. + int64_t nMinPingUsecTime; // Whether a ping is requested. bool fPingQueued; diff --git a/src/netbase.cpp b/src/netbase.cpp index 7a87d125c2..c3d56d9184 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -1311,17 +1311,57 @@ bool CSubNet::Match(const CNetAddr &addr) const return true; } +static inline int NetmaskBits(uint8_t x) +{ + switch(x) { + case 0x00: return 0; break; + case 0x80: return 1; break; + case 0xc0: return 2; break; + case 0xe0: return 3; break; + case 0xf0: return 4; break; + case 0xf8: return 5; break; + case 0xfc: return 6; break; + case 0xfe: return 7; break; + case 0xff: return 8; break; + default: return -1; break; + } +} + std::string CSubNet::ToString() const { + /* Parse binary 1{n}0{N-n} to see if mask can be represented as /n */ + int cidr = 0; + bool valid_cidr = true; + int n = network.IsIPv4() ? 12 : 0; + for (; n < 16 && netmask[n] == 0xff; ++n) + cidr += 8; + if (n < 16) { + int bits = NetmaskBits(netmask[n]); + if (bits < 0) + valid_cidr = false; + else + cidr += bits; + ++n; + } + for (; n < 16 && valid_cidr; ++n) + if (netmask[n] != 0x00) + valid_cidr = false; + + /* Format output */ std::string strNetmask; - if (network.IsIPv4()) - strNetmask = strprintf("%u.%u.%u.%u", netmask[12], netmask[13], netmask[14], netmask[15]); - else - strNetmask = strprintf("%x:%x:%x:%x:%x:%x:%x:%x", - netmask[0] << 8 | netmask[1], netmask[2] << 8 | netmask[3], - netmask[4] << 8 | netmask[5], netmask[6] << 8 | netmask[7], - netmask[8] << 8 | netmask[9], netmask[10] << 8 | netmask[11], - netmask[12] << 8 | netmask[13], netmask[14] << 8 | netmask[15]); + if (valid_cidr) { + strNetmask = strprintf("%u", cidr); + } else { + if (network.IsIPv4()) + strNetmask = strprintf("%u.%u.%u.%u", netmask[12], netmask[13], netmask[14], netmask[15]); + else + strNetmask = strprintf("%x:%x:%x:%x:%x:%x:%x:%x", + netmask[0] << 8 | netmask[1], netmask[2] << 8 | netmask[3], + netmask[4] << 8 | netmask[5], netmask[6] << 8 | netmask[7], + netmask[8] << 8 | netmask[9], netmask[10] << 8 | netmask[11], + netmask[12] << 8 | netmask[13], netmask[14] << 8 | netmask[15]); + } + return network.ToString() + "/" + strNetmask; } diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 5b9c13d870..7a58074d24 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -15,7 +15,7 @@ uint256 CBlockHeader::GetHash() const return SerializeHash(*this); } -uint256 CBlock::BuildMerkleTree(bool* fMutated) const +uint256 CBlock::ComputeMerkleRoot(bool* fMutated) const { /* WARNING! If you're reading this because you're learning about crypto and/or designing a new system that will use merkle trees, keep in mind @@ -52,7 +52,7 @@ uint256 CBlock::BuildMerkleTree(bool* fMutated) const known ways of changing the transactions without affecting the merkle root. */ - vMerkleTree.clear(); + std::vector<uint256> vMerkleTree; vMerkleTree.reserve(vtx.size() * 2 + 16); // Safe upper bound for the number of total nodes. for (std::vector<CTransaction>::const_iterator it(vtx.begin()); it != vtx.end(); ++it) vMerkleTree.push_back(it->GetHash()); @@ -78,37 +78,6 @@ uint256 CBlock::BuildMerkleTree(bool* fMutated) const return (vMerkleTree.empty() ? uint256() : vMerkleTree.back()); } -std::vector<uint256> CBlock::GetMerkleBranch(int nIndex) const -{ - if (vMerkleTree.empty()) - BuildMerkleTree(); - std::vector<uint256> vMerkleBranch; - int j = 0; - for (int nSize = vtx.size(); nSize > 1; nSize = (nSize + 1) / 2) - { - int i = std::min(nIndex^1, nSize-1); - vMerkleBranch.push_back(vMerkleTree[j+i]); - nIndex >>= 1; - j += nSize; - } - return vMerkleBranch; -} - -uint256 CBlock::CheckMerkleBranch(uint256 hash, const std::vector<uint256>& vMerkleBranch, int nIndex) -{ - if (nIndex == -1) - return uint256(); - for (std::vector<uint256>::const_iterator it(vMerkleBranch.begin()); it != vMerkleBranch.end(); ++it) - { - if (nIndex & 1) - hash = Hash(BEGIN(*it), END(*it), BEGIN(hash), END(hash)); - else - hash = Hash(BEGIN(hash), END(hash), BEGIN(*it), END(*it)); - nIndex >>= 1; - } - return hash; -} - std::string CBlock::ToString() const { std::stringstream s; @@ -123,9 +92,5 @@ std::string CBlock::ToString() const { s << " " << vtx[i].ToString() << "\n"; } - s << " vMerkleTree: "; - for (unsigned int i = 0; i < vMerkleTree.size(); i++) - s << " " << vMerkleTree[i].ToString(); - s << "\n"; return s.str(); } diff --git a/src/primitives/block.h b/src/primitives/block.h index 59f46deb1c..86106098f5 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -78,7 +78,7 @@ public: std::vector<CTransaction> vtx; // memory only - mutable std::vector<uint256> vMerkleTree; + mutable bool fChecked; CBlock() { @@ -103,7 +103,7 @@ public: { CBlockHeader::SetNull(); vtx.clear(); - vMerkleTree.clear(); + fChecked = false; } CBlockHeader GetBlockHeader() const @@ -118,14 +118,12 @@ public: return block; } - // Build the in-memory merkle tree for this block and return the merkle root. + // Build the merkle tree for this block and return the merkle root. // If non-NULL, *mutated is set to whether mutation was detected in the merkle // tree (a duplication of transactions in the block leading to an identical // merkle root). - uint256 BuildMerkleTree(bool* mutated = NULL) const; + uint256 ComputeMerkleRoot(bool* mutated = NULL) const; - std::vector<uint256> GetMerkleBranch(int nIndex) const; - static uint256 CheckMerkleBranch(uint256 hash, const std::vector<uint256>& vMerkleBranch, int nIndex); std::string ToString() const; }; diff --git a/src/protocol.h b/src/protocol.h index b5e65032a2..50aeaf44ba 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -75,6 +75,10 @@ enum { // Bitcoin Core does not support this but a patch set called Bitcoin XT does. // See BIP 64 for details on how this is implemented. NODE_GETUTXO = (1 << 1), + // NODE_BLOOM means the node is capable and willing to handle bloom-filtered connections. + // Bitcoin Core nodes used to support this by default, without advertising this bit, + // but no longer do as of protocol version 70011 (= NO_BLOOM_VERSION) + NODE_BLOOM = (1 << 2), // Bits 24-31 are reserved for temporary experiments. Just pick a bit that // isn't getting used, or one not being used much, and notify the diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp new file mode 100644 index 0000000000..33792af5ba --- /dev/null +++ b/src/qt/bantablemodel.cpp @@ -0,0 +1,181 @@ +// Copyright (c) 2011-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bantablemodel.h" + +#include "clientmodel.h" +#include "guiconstants.h" +#include "guiutil.h" + +#include "sync.h" +#include "utiltime.h" + +#include <QDebug> +#include <QList> + +bool BannedNodeLessThan::operator()(const CCombinedBan& left, const CCombinedBan& right) const +{ + const CCombinedBan* pLeft = &left; + const CCombinedBan* pRight = &right; + + if (order == Qt::DescendingOrder) + std::swap(pLeft, pRight); + + switch(column) + { + case BanTableModel::Address: + return pLeft->subnet.ToString().compare(pRight->subnet.ToString()) < 0; + case BanTableModel::Bantime: + return pLeft->banEntry.nBanUntil < pRight->banEntry.nBanUntil; + } + + return false; +} + +// private implementation +class BanTablePriv +{ +public: + /** Local cache of peer information */ + QList<CCombinedBan> cachedBanlist; + /** Column to sort nodes by */ + int sortColumn; + /** Order (ascending or descending) to sort nodes by */ + Qt::SortOrder sortOrder; + + /** Pull a full list of banned nodes from CNode into our cache */ + void refreshBanlist() + { + banmap_t banMap; + CNode::GetBanned(banMap); + + cachedBanlist.clear(); +#if QT_VERSION >= 0x040700 + cachedBanlist.reserve(banMap.size()); +#endif + for (banmap_t::iterator it = banMap.begin(); it != banMap.end(); it++) + { + CCombinedBan banEntry; + banEntry.subnet = (*it).first; + banEntry.banEntry = (*it).second; + cachedBanlist.append(banEntry); + } + + if (sortColumn >= 0) + // sort cachedBanlist (use stable sort to prevent rows jumping around unneceesarily) + qStableSort(cachedBanlist.begin(), cachedBanlist.end(), BannedNodeLessThan(sortColumn, sortOrder)); + } + + int size() const + { + return cachedBanlist.size(); + } + + CCombinedBan *index(int idx) + { + if (idx >= 0 && idx < cachedBanlist.size()) + return &cachedBanlist[idx]; + + return 0; + } +}; + +BanTableModel::BanTableModel(ClientModel *parent) : + QAbstractTableModel(parent), + clientModel(parent) +{ + columns << tr("IP/Netmask") << tr("Banned Until"); + priv = new BanTablePriv(); + // default to unsorted + priv->sortColumn = -1; + + // load initial data + refresh(); +} + +int BanTableModel::rowCount(const QModelIndex &parent) const +{ + Q_UNUSED(parent); + return priv->size(); +} + +int BanTableModel::columnCount(const QModelIndex &parent) const +{ + Q_UNUSED(parent); + return columns.length();; +} + +QVariant BanTableModel::data(const QModelIndex &index, int role) const +{ + if(!index.isValid()) + return QVariant(); + + CCombinedBan *rec = static_cast<CCombinedBan*>(index.internalPointer()); + + if (role == Qt::DisplayRole) { + switch(index.column()) + { + case Address: + return QString::fromStdString(rec->subnet.ToString()); + case Bantime: + QDateTime date = QDateTime::fromMSecsSinceEpoch(0); + date = date.addSecs(rec->banEntry.nBanUntil); + return date.toString(Qt::SystemLocaleLongDate); + } + } + + return QVariant(); +} + +QVariant BanTableModel::headerData(int section, Qt::Orientation orientation, int role) const +{ + if(orientation == Qt::Horizontal) + { + if(role == Qt::DisplayRole && section < columns.size()) + { + return columns[section]; + } + } + return QVariant(); +} + +Qt::ItemFlags BanTableModel::flags(const QModelIndex &index) const +{ + if(!index.isValid()) + return 0; + + Qt::ItemFlags retval = Qt::ItemIsSelectable | Qt::ItemIsEnabled; + return retval; +} + +QModelIndex BanTableModel::index(int row, int column, const QModelIndex &parent) const +{ + Q_UNUSED(parent); + CCombinedBan *data = priv->index(row); + + if (data) + return createIndex(row, column, data); + return QModelIndex(); +} + +void BanTableModel::refresh() +{ + Q_EMIT layoutAboutToBeChanged(); + priv->refreshBanlist(); + Q_EMIT layoutChanged(); +} + +void BanTableModel::sort(int column, Qt::SortOrder order) +{ + priv->sortColumn = column; + priv->sortOrder = order; + refresh(); +} + +bool BanTableModel::shouldShow() +{ + if (priv->size() > 0) + return true; + return false; +}
\ No newline at end of file diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h new file mode 100644 index 0000000000..c21dd04e31 --- /dev/null +++ b/src/qt/bantablemodel.h @@ -0,0 +1,72 @@ +// Copyright (c) 2011-2013 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_BANTABLEMODEL_H +#define BITCOIN_QT_BANTABLEMODEL_H + +#include "net.h" + +#include <QAbstractTableModel> +#include <QStringList> + +class ClientModel; +class BanTablePriv; + +struct CCombinedBan { + CSubNet subnet; + CBanEntry banEntry; +}; + +class BannedNodeLessThan +{ +public: + BannedNodeLessThan(int nColumn, Qt::SortOrder fOrder) : + column(nColumn), order(fOrder) {} + bool operator()(const CCombinedBan& left, const CCombinedBan& right) const; + +private: + int column; + Qt::SortOrder order; +}; + +/** + Qt model providing information about connected peers, similar to the + "getpeerinfo" RPC call. Used by the rpc console UI. + */ +class BanTableModel : public QAbstractTableModel +{ + Q_OBJECT + +public: + explicit BanTableModel(ClientModel *parent = 0); + void startAutoRefresh(); + void stopAutoRefresh(); + + enum ColumnIndex { + Address = 0, + Bantime = 1 + }; + + /** @name Methods overridden from QAbstractTableModel + @{*/ + int rowCount(const QModelIndex &parent) const; + int columnCount(const QModelIndex &parent) const; + QVariant data(const QModelIndex &index, int role) const; + QVariant headerData(int section, Qt::Orientation orientation, int role) const; + QModelIndex index(int row, int column, const QModelIndex &parent) const; + Qt::ItemFlags flags(const QModelIndex &index) const; + void sort(int column, Qt::SortOrder order); + bool shouldShow(); + /*@}*/ + +public Q_SLOTS: + void refresh(); + +private: + ClientModel *clientModel; + QStringList columns; + BanTablePriv *priv; +}; + +#endif // BITCOIN_QT_BANTABLEMODEL_H diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 1da2d3e344..ea7f86d18e 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -266,13 +266,6 @@ void BitcoinCore::initialize() { qDebug() << __func__ << ": Running AppInit2 in thread"; int rv = AppInit2(threadGroup, scheduler); - if(rv) - { - /* Start a dummy RPC thread if no RPC thread is active yet - * to handle timeouts. - */ - StartDummyRPCThread(); - } Q_EMIT initializeResult(rv); } catch (const std::exception& e) { handleRunawayException(&e); @@ -286,7 +279,7 @@ void BitcoinCore::shutdown() try { qDebug() << __func__ << ": Running Shutdown in thread"; - threadGroup.interrupt_all(); + Interrupt(threadGroup); threadGroup.join_all(); Shutdown(); qDebug() << __func__ << ": Shutdown finished"; diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 1d3f7762ab..d930d15953 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -97,6 +97,7 @@ BitcoinGUI::BitcoinGUI(const PlatformStyle *platformStyle, const NetworkStyle *n trayIconMenu(0), notificator(0), rpcConsole(0), + helpMessageDialog(0), prevBlocks(0), spinnerFrame(0), platformStyle(platformStyle) @@ -132,6 +133,7 @@ BitcoinGUI::BitcoinGUI(const PlatformStyle *platformStyle, const NetworkStyle *n #endif rpcConsole = new RPCConsole(platformStyle, 0); + helpMessageDialog = new HelpMessageDialog(this, false); #ifdef ENABLE_WALLET if(enableWallet) { @@ -213,11 +215,6 @@ BitcoinGUI::BitcoinGUI(const PlatformStyle *platformStyle, const NetworkStyle *n statusBar()->addWidget(progressBar); statusBar()->addPermanentWidget(frameBlocks); - connect(openRPCConsoleAction, SIGNAL(triggered()), rpcConsole, SLOT(show())); - - // prevents an open debug window from becoming stuck/unusable on client shutdown - connect(quitAction, SIGNAL(triggered()), rpcConsole, SLOT(hide())); - // Install event filter to be able to catch status tip events (QEvent::StatusTip) this->installEventFilter(this); @@ -350,6 +347,10 @@ void BitcoinGUI::createActions() connect(optionsAction, SIGNAL(triggered()), this, SLOT(optionsClicked())); connect(toggleHideAction, SIGNAL(triggered()), this, SLOT(toggleHidden())); connect(showHelpMessageAction, SIGNAL(triggered()), this, SLOT(showHelpMessageClicked())); + connect(openRPCConsoleAction, SIGNAL(triggered()), this, SLOT(showDebugWindow())); + // prevents an open debug window from becoming stuck/unusable on client shutdown + connect(quitAction, SIGNAL(triggered()), rpcConsole, SLOT(hide())); + #ifdef ENABLE_WALLET if(walletFrame) { @@ -588,11 +589,17 @@ void BitcoinGUI::aboutClicked() dlg.exec(); } +void BitcoinGUI::showDebugWindow() +{ + rpcConsole->showNormal(); + rpcConsole->show(); + rpcConsole->raise(); + rpcConsole->activateWindow(); +} + void BitcoinGUI::showHelpMessageClicked() { - HelpMessageDialog *help = new HelpMessageDialog(this, false); - help->setAttribute(Qt::WA_DeleteOnClose); - help->show(); + helpMessageDialog->show(); } #ifdef ENABLE_WALLET diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index dd0d4bb0e2..717f2bd12d 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -28,6 +28,7 @@ class SendCoinsRecipient; class UnitDisplayStatusBarControl; class WalletFrame; class WalletModel; +class HelpMessageDialog; class CWallet; @@ -113,6 +114,7 @@ private: QMenu *trayIconMenu; Notificator *notificator; RPCConsole *rpcConsole; + HelpMessageDialog *helpMessageDialog; /** Keep track of previous number of blocks, to detect progress */ int prevBlocks; @@ -194,6 +196,8 @@ private Q_SLOTS: void optionsClicked(); /** Show about dialog */ void aboutClicked(); + /** Show debug window */ + void showDebugWindow(); /** Show help message dialog */ void showHelpMessageClicked(); #ifndef Q_OS_MAC diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 97d6711560..0900a35cc4 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -4,6 +4,7 @@ #include "clientmodel.h" +#include "bantablemodel.h" #include "guiconstants.h" #include "peertablemodel.h" @@ -26,6 +27,7 @@ ClientModel::ClientModel(OptionsModel *optionsModel, QObject *parent) : QObject(parent), optionsModel(optionsModel), peerTableModel(0), + banTableModel(0), cachedNumBlocks(0), cachedBlockDate(QDateTime()), cachedReindexing(0), @@ -33,6 +35,7 @@ ClientModel::ClientModel(OptionsModel *optionsModel, QObject *parent) : pollTimer(0) { peerTableModel = new PeerTableModel(this); + banTableModel = new BanTableModel(this); pollTimer = new QTimer(this); connect(pollTimer, SIGNAL(timeout()), this, SLOT(updateTimer())); pollTimer->start(MODEL_UPDATE_DELAY); @@ -176,6 +179,11 @@ PeerTableModel *ClientModel::getPeerTableModel() return peerTableModel; } +BanTableModel *ClientModel::getBanTableModel() +{ + return banTableModel; +} + QString ClientModel::formatFullVersion() const { return QString::fromStdString(FormatFullVersion()); @@ -206,6 +214,11 @@ QString ClientModel::formatClientStartupTime() const return QDateTime::fromTime_t(nClientStartupTime).toString(); } +void ClientModel::updateBanlist() +{ + banTableModel->refresh(); +} + // Handlers for core signals static void ShowProgress(ClientModel *clientmodel, const std::string &title, int nProgress) { @@ -230,12 +243,19 @@ static void NotifyAlertChanged(ClientModel *clientmodel, const uint256 &hash, Ch Q_ARG(int, status)); } +static void BannedListChanged(ClientModel *clientmodel) +{ + qDebug() << QString("%1: Requesting update for peer banlist").arg(__func__); + QMetaObject::invokeMethod(clientmodel, "updateBanlist", Qt::QueuedConnection); +} + void ClientModel::subscribeToCoreSignals() { // Connect signals to client uiInterface.ShowProgress.connect(boost::bind(ShowProgress, this, _1, _2)); uiInterface.NotifyNumConnectionsChanged.connect(boost::bind(NotifyNumConnectionsChanged, this, _1)); uiInterface.NotifyAlertChanged.connect(boost::bind(NotifyAlertChanged, this, _1, _2)); + uiInterface.BannedListChanged.connect(boost::bind(BannedListChanged, this)); } void ClientModel::unsubscribeFromCoreSignals() @@ -244,4 +264,5 @@ void ClientModel::unsubscribeFromCoreSignals() uiInterface.ShowProgress.disconnect(boost::bind(ShowProgress, this, _1, _2)); uiInterface.NotifyNumConnectionsChanged.disconnect(boost::bind(NotifyNumConnectionsChanged, this, _1)); uiInterface.NotifyAlertChanged.disconnect(boost::bind(NotifyAlertChanged, this, _1, _2)); + uiInterface.BannedListChanged.disconnect(boost::bind(BannedListChanged, this)); } diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index ca2da3dde0..627bdf862d 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -9,6 +9,7 @@ #include <QDateTime> class AddressTableModel; +class BanTableModel; class OptionsModel; class PeerTableModel; class TransactionTableModel; @@ -44,6 +45,7 @@ public: OptionsModel *getOptionsModel(); PeerTableModel *getPeerTableModel(); + BanTableModel *getBanTableModel(); //! Return number of connections, default is in- and outbound (total) int getNumConnections(unsigned int flags = CONNECTIONS_ALL) const; @@ -72,6 +74,7 @@ public: private: OptionsModel *optionsModel; PeerTableModel *peerTableModel; + BanTableModel *banTableModel; int cachedNumBlocks; QDateTime cachedBlockDate; @@ -99,6 +102,7 @@ public Q_SLOTS: void updateTimer(); void updateNumConnections(int numConnections); void updateAlert(const QString &hash, int status); + void updateBanlist(); }; #endif // BITCOIN_QT_CLIENTMODEL_H diff --git a/src/qt/forms/rpcconsole.ui b/src/qt/forms/debugwindow.ui index e8d9a958ad..4117da57f5 100644 --- a/src/qt/forms/rpcconsole.ui +++ b/src/qt/forms/debugwindow.ui @@ -713,17 +713,85 @@ </attribute> <layout class="QGridLayout" name="gridLayout_2"> <item row="0" column="0" rowspan="2"> - <widget class="QTableView" name="peerWidget"> - <property name="horizontalScrollBarPolicy"> - <enum>Qt::ScrollBarAsNeeded</enum> - </property> - <property name="sortingEnabled"> - <bool>true</bool> + <layout class="QVBoxLayout" name="verticalLayout_101"> + <property name="spacing"> + <number>0</number> </property> - <attribute name="horizontalHeaderHighlightSections"> - <bool>false</bool> - </attribute> - </widget> + <item> + <widget class="QTableView" name="peerWidget"> + <property name="horizontalScrollBarPolicy"> + <enum>Qt::ScrollBarAsNeeded</enum> + </property> + <property name="sortingEnabled"> + <bool>true</bool> + </property> + <attribute name="horizontalHeaderHighlightSections"> + <bool>false</bool> + </attribute> + </widget> + </item> + <item> + <widget class="QLabel" name="banHeading"> + <property name="sizePolicy"> + <sizepolicy hsizetype="Preferred" vsizetype="Minimum"> + <horstretch>0</horstretch> + <verstretch>0</verstretch> + </sizepolicy> + </property> + <property name="minimumSize"> + <size> + <width>300</width> + <height>32</height> + </size> + </property> + <property name="maximumSize"> + <size> + <width>16777215</width> + <height>32</height> + </size> + </property> + <property name="font"> + <font> + <pointsize>12</pointsize> + </font> + </property> + <property name="cursor"> + <cursorShape>IBeamCursor</cursorShape> + </property> + <property name="text"> + <string>Banned peers</string> + </property> + <property name="alignment"> + <set>Qt::AlignBottom|Qt::AlignLeading|Qt::AlignLeft</set> + </property> + <property name="wordWrap"> + <bool>true</bool> + </property> + <property name="textInteractionFlags"> + <set>Qt::NoTextInteraction</set> + </property> + </widget> + </item> + <item> + <widget class="QTableView" name="banlistWidget"> + <property name="sizePolicy"> + <sizepolicy hsizetype="Expanding" vsizetype="Expanding"> + <horstretch>0</horstretch> + <verstretch>0</verstretch> + </sizepolicy> + </property> + <property name="horizontalScrollBarPolicy"> + <enum>Qt::ScrollBarAsNeeded</enum> + </property> + <property name="sortingEnabled"> + <bool>true</bool> + </property> + <attribute name="horizontalHeaderHighlightSections"> + <bool>false</bool> + </attribute> + </widget> + </item> + </layout> </item> <item row="0" column="1"> <widget class="QLabel" name="peerHeading"> diff --git a/src/qt/guiconstants.h b/src/qt/guiconstants.h index a0a2993ea3..7d3e48ff32 100644 --- a/src/qt/guiconstants.h +++ b/src/qt/guiconstants.h @@ -42,7 +42,7 @@ static const int MAX_URI_LENGTH = 255; #define EXPORT_IMAGE_SIZE 256 /* Number of frames in spinner animation */ -#define SPINNER_FRAMES 35 +#define SPINNER_FRAMES 36 #define QAPP_ORG_NAME "Bitcoin" #define QAPP_ORG_DOMAIN "bitcoin.org" diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 550dbacf93..8917f77f22 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -404,7 +404,7 @@ void SubstituteFonts(const QString& language) { #if defined(Q_OS_MAC) // Background: -// OSX's default font changed in 10.9 and QT is unable to find it with its +// OSX's default font changed in 10.9 and Qt is unable to find it with its // usual fallback methods when building against the 10.7 sdk or lower. // The 10.8 SDK added a function to let it find the correct fallback font. // If this fallback is not properly loaded, some characters may fail to diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index 5cc4d00dbf..31a6d65a8d 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -509,12 +509,7 @@ bool PaymentServer::readPaymentRequestFromFile(const QString& filename, PaymentR } // BIP70 DoS protection - if (f.size() > BIP70_MAX_PAYMENTREQUEST_SIZE) { - qWarning() << QString("PaymentServer::%1: Payment request %2 is too large (%3 bytes, allowed %4 bytes).") - .arg(__func__) - .arg(filename) - .arg(f.size()) - .arg(BIP70_MAX_PAYMENTREQUEST_SIZE); + if (!verifySize(f.size())) { return false; } @@ -685,14 +680,13 @@ void PaymentServer::netRequestFinished(QNetworkReply* reply) reply->deleteLater(); // BIP70 DoS protection - if (reply->size() > BIP70_MAX_PAYMENTREQUEST_SIZE) { - QString msg = tr("Payment request %1 is too large (%2 bytes, allowed %3 bytes).") - .arg(reply->request().url().toString()) - .arg(reply->size()) - .arg(BIP70_MAX_PAYMENTREQUEST_SIZE); - - qWarning() << QString("PaymentServer::%1:").arg(__func__) << msg; - Q_EMIT message(tr("Payment request DoS protection"), msg, CClientUIInterface::MSG_ERROR); + if (!verifySize(reply->size())) { + Q_EMIT message(tr("Payment request rejected"), + tr("Payment request %1 is too large (%2 bytes, allowed %3 bytes).") + .arg(reply->request().url().toString()) + .arg(reply->size()) + .arg(BIP70_MAX_PAYMENTREQUEST_SIZE), + CClientUIInterface::MSG_ERROR); return; } @@ -790,6 +784,18 @@ bool PaymentServer::verifyExpired(const payments::PaymentDetails& requestDetails return fVerified; } +bool PaymentServer::verifySize(qint64 requestSize) +{ + bool fVerified = (requestSize <= BIP70_MAX_PAYMENTREQUEST_SIZE); + if (!fVerified) { + qWarning() << QString("PaymentServer::%1: Payment request too large (%2 bytes, allowed %3 bytes).") + .arg(__func__) + .arg(requestSize) + .arg(BIP70_MAX_PAYMENTREQUEST_SIZE); + } + return fVerified; +} + bool PaymentServer::verifyAmount(const CAmount& requestAmount) { bool fVerified = MoneyRange(requestAmount); diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 5df0a14cf7..fa120a435c 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -88,13 +88,12 @@ public: // OptionsModel is used for getting proxy settings and display unit void setOptionsModel(OptionsModel *optionsModel); - // This is now public, because we use it in paymentservertests.cpp - static bool readPaymentRequestFromFile(const QString& filename, PaymentRequestPlus& request); - // Verify that the payment request network matches the client network static bool verifyNetwork(const payments::PaymentDetails& requestDetails); // Verify if the payment request is expired static bool verifyExpired(const payments::PaymentDetails& requestDetails); + // Verify the payment request size is valid as per BIP70 + static bool verifySize(qint64 requestSize); // Verify the payment request amount is valid static bool verifyAmount(const CAmount& requestAmount); @@ -131,6 +130,7 @@ protected: bool eventFilter(QObject *object, QEvent *event); private: + static bool readPaymentRequestFromFile(const QString& filename, PaymentRequestPlus& request); bool processPaymentRequest(const PaymentRequestPlus& request, SendCoinsRecipient& recipient); void fetchRequest(const QUrl& url); diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 85339166b0..770a860544 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -8,7 +8,6 @@ #include "guiconstants.h" #include "guiutil.h" -#include "net.h" #include "sync.h" #include <QDebug> @@ -96,18 +95,17 @@ public: mapNodeRows.insert(std::pair<NodeId, int>(stats.nodeStats.nodeid, row++)); } - int size() + int size() const { return cachedNodeStats.size(); } CNodeCombinedStats *index(int idx) { - if(idx >= 0 && idx < cachedNodeStats.size()) { + if (idx >= 0 && idx < cachedNodeStats.size()) return &cachedNodeStats[idx]; - } else { - return 0; - } + + return 0; } }; @@ -171,7 +169,7 @@ QVariant PeerTableModel::data(const QModelIndex &index, int role) const } } else if (role == Qt::TextAlignmentRole) { if (index.column() == Ping) - return (int)(Qt::AlignRight | Qt::AlignVCenter); + return (QVariant)(Qt::AlignRight | Qt::AlignVCenter); } return QVariant(); @@ -204,13 +202,8 @@ QModelIndex PeerTableModel::index(int row, int column, const QModelIndex &parent CNodeCombinedStats *data = priv->index(row); if (data) - { return createIndex(row, column, data); - } - else - { - return QModelIndex(); - } + return QModelIndex(); } const CNodeCombinedStats *PeerTableModel::getNodeStats(int idx) diff --git a/src/qt/res/movies/makespinner.sh b/src/qt/res/movies/makespinner.sh index 625fb17173..a4c2fddbbf 100755 --- a/src/qt/res/movies/makespinner.sh +++ b/src/qt/res/movies/makespinner.sh @@ -1,6 +1,7 @@ -for i in {1..35} +FRAMEDIR=$(dirname $0) +for i in {0..35} do - value=$(printf "%03d" $i) + frame=$(printf "%03d" $i) angle=$(($i * 10)) - convert spinner-000.png -background "rgba(0,0,0,0.0)" -distort SRT $angle spinner-$value.png + convert $FRAMEDIR/../src/spinner.png -background "rgba(0,0,0,0.0)" -distort SRT $angle $FRAMEDIR/spinner-$frame.png done diff --git a/src/qt/res/movies/spinner-000.png b/src/qt/res/movies/spinner-000.png Binary files differindex 1e92d859da..0dc48d0d8c 100644 --- a/src/qt/res/movies/spinner-000.png +++ b/src/qt/res/movies/spinner-000.png diff --git a/src/qt/res/spinner.png b/src/qt/res/src/spinner.png Binary files differindex b296a58481..b296a58481 100644 --- a/src/qt/res/spinner.png +++ b/src/qt/res/src/spinner.png diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index d9d4f1d0ed..72a3023c9a 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -3,12 +3,13 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "rpcconsole.h" -#include "ui_rpcconsole.h" +#include "ui_debugwindow.h" +#include "bantablemodel.h" #include "clientmodel.h" #include "guiutil.h" -#include "peertablemodel.h" #include "platformstyle.h" +#include "bantablemodel.h" #include "chainparams.h" #include "rpcserver.h" @@ -26,8 +27,10 @@ #include <QKeyEvent> #include <QMenu> #include <QScrollBar> +#include <QSignalMapper> #include <QThread> #include <QTime> +#include <QTimer> #if QT_VERSION < 0x050000 #include <QUrl> @@ -66,6 +69,40 @@ Q_SIGNALS: void reply(int category, const QString &command); }; +/** Class for handling RPC timers + * (used for e.g. re-locking the wallet after a timeout) + */ +class QtRPCTimerBase: public QObject, public RPCTimerBase +{ + Q_OBJECT +public: + QtRPCTimerBase(boost::function<void(void)>& func, int64_t millis): + func(func) + { + timer.setSingleShot(true); + connect(&timer, SIGNAL(timeout()), this, SLOT(timeout())); + timer.start(millis); + } + ~QtRPCTimerBase() {} +private Q_SLOTS: + void timeout() { func(); } +private: + QTimer timer; + boost::function<void(void)> func; +}; + +class QtRPCTimerInterface: public RPCTimerInterface +{ +public: + ~QtRPCTimerInterface() {} + const char *Name() { return "Qt"; } + RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) + { + return new QtRPCTimerBase(func, millis); + } +}; + + #include "rpcconsole.moc" /** @@ -206,8 +243,9 @@ RPCConsole::RPCConsole(const PlatformStyle *platformStyle, QWidget *parent) : clientModel(0), historyPtr(0), cachedNodeid(-1), - contextMenu(0), - platformStyle(platformStyle) + platformStyle(platformStyle), + peersTableContextMenu(0), + banTableContextMenu(0) { ui->setupUi(this); GUIUtil::restoreWindowGeometry("nRPCConsoleWindow", this->size(), this); @@ -232,6 +270,9 @@ RPCConsole::RPCConsole(const PlatformStyle *platformStyle, QWidget *parent) : ui->label_berkeleyDBVersion->hide(); ui->berkeleyDBVersion->hide(); #endif + // Register RPC timer interface + rpcTimerInterface = new QtRPCTimerInterface(); + RPCRegisterTimerInterface(rpcTimerInterface); startExecutor(); setTrafficGraphRange(INITIAL_TRAFFIC_GRAPH_MINS); @@ -246,6 +287,8 @@ RPCConsole::~RPCConsole() { GUIUtil::saveWindowGeometry("nRPCConsoleWindow", this); Q_EMIT stopExecutor(); + RPCUnregisterTimerInterface(rpcTimerInterface); + delete rpcTimerInterface; delete ui; } @@ -289,8 +332,7 @@ void RPCConsole::setClientModel(ClientModel *model) { clientModel = model; ui->trafficGraph->setClientModel(model); - if(model) - { + if (model && clientModel->getPeerTableModel() && clientModel->getBanTableModel()) { // Keep up to date with client setNumConnections(model->getNumConnections()); connect(model, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int))); @@ -311,23 +353,75 @@ void RPCConsole::setClientModel(ClientModel *model) ui->peerWidget->setColumnWidth(PeerTableModel::Address, ADDRESS_COLUMN_WIDTH); ui->peerWidget->setColumnWidth(PeerTableModel::Subversion, SUBVERSION_COLUMN_WIDTH); ui->peerWidget->setColumnWidth(PeerTableModel::Ping, PING_COLUMN_WIDTH); + ui->peerWidget->horizontalHeader()->setStretchLastSection(true); - // create context menu actions + // create peer table context menu actions QAction* disconnectAction = new QAction(tr("&Disconnect Node"), this); - - // create context menu - contextMenu = new QMenu(); - contextMenu->addAction(disconnectAction); - - // context menu signals - connect(ui->peerWidget, SIGNAL(customContextMenuRequested(const QPoint&)), this, SLOT(showMenu(const QPoint&))); + QAction* banAction1h = new QAction(tr("Ban Node for") + " " + tr("1 &hour"), this); + QAction* banAction24h = new QAction(tr("Ban Node for") + " " + tr("1 &day"), this); + QAction* banAction7d = new QAction(tr("Ban Node for") + " " + tr("1 &week"), this); + QAction* banAction365d = new QAction(tr("Ban Node for") + " " + tr("1 &year"), this); + + // create peer table context menu + peersTableContextMenu = new QMenu(); + peersTableContextMenu->addAction(disconnectAction); + peersTableContextMenu->addAction(banAction1h); + peersTableContextMenu->addAction(banAction24h); + peersTableContextMenu->addAction(banAction7d); + peersTableContextMenu->addAction(banAction365d); + + // Add a signal mapping to allow dynamic context menu arguments. + // We need to use int (instead of int64_t), because signal mapper only supports + // int or objects, which is okay because max bantime (1 year) is < int_max. + QSignalMapper* signalMapper = new QSignalMapper(this); + signalMapper->setMapping(banAction1h, 60*60); + signalMapper->setMapping(banAction24h, 60*60*24); + signalMapper->setMapping(banAction7d, 60*60*24*7); + signalMapper->setMapping(banAction365d, 60*60*24*365); + connect(banAction1h, SIGNAL(triggered()), signalMapper, SLOT(map())); + connect(banAction24h, SIGNAL(triggered()), signalMapper, SLOT(map())); + connect(banAction7d, SIGNAL(triggered()), signalMapper, SLOT(map())); + connect(banAction365d, SIGNAL(triggered()), signalMapper, SLOT(map())); + connect(signalMapper, SIGNAL(mapped(int)), this, SLOT(banSelectedNode(int))); + + // peer table context menu signals + connect(ui->peerWidget, SIGNAL(customContextMenuRequested(const QPoint&)), this, SLOT(showPeersTableContextMenu(const QPoint&))); connect(disconnectAction, SIGNAL(triggered()), this, SLOT(disconnectSelectedNode())); - // connect the peerWidget selection model to our peerSelected() handler + // peer table signal handling - update peer details when selecting new node connect(ui->peerWidget->selectionModel(), SIGNAL(selectionChanged(const QItemSelection &, const QItemSelection &)), - this, SLOT(peerSelected(const QItemSelection &, const QItemSelection &))); + this, SLOT(peerSelected(const QItemSelection &, const QItemSelection &))); + // peer table signal handling - update peer details when new nodes are added to the model connect(model->getPeerTableModel(), SIGNAL(layoutChanged()), this, SLOT(peerLayoutChanged())); + // set up ban table + ui->banlistWidget->setModel(model->getBanTableModel()); + ui->banlistWidget->verticalHeader()->hide(); + ui->banlistWidget->setEditTriggers(QAbstractItemView::NoEditTriggers); + ui->banlistWidget->setSelectionBehavior(QAbstractItemView::SelectRows); + ui->banlistWidget->setSelectionMode(QAbstractItemView::SingleSelection); + ui->banlistWidget->setContextMenuPolicy(Qt::CustomContextMenu); + ui->banlistWidget->setColumnWidth(BanTableModel::Address, BANSUBNET_COLUMN_WIDTH); + ui->banlistWidget->setColumnWidth(BanTableModel::Bantime, BANTIME_COLUMN_WIDTH); + ui->banlistWidget->horizontalHeader()->setStretchLastSection(true); + + // create ban table context menu action + QAction* unbanAction = new QAction(tr("&Unban Node"), this); + + // create ban table context menu + banTableContextMenu = new QMenu(); + banTableContextMenu->addAction(unbanAction); + + // ban table context menu signals + connect(ui->banlistWidget, SIGNAL(customContextMenuRequested(const QPoint&)), this, SLOT(showBanTableContextMenu(const QPoint&))); + connect(unbanAction, SIGNAL(triggered()), this, SLOT(unbanSelectedNode())); + + // ban table signal handling - clear peer details when clicking a peer in the ban table + connect(ui->banlistWidget, SIGNAL(clicked(const QModelIndex&)), this, SLOT(clearSelectedNode())); + // ban table signal handling - ensure ban table is shown or hidden (if empty) + connect(model->getBanTableModel(), SIGNAL(layoutChanged()), this, SLOT(showOrHideBanTableIfRequired())); + showOrHideBanTableIfRequired(); + // Provide initial values ui->clientVersion->setText(model->formatFullVersion()); ui->clientUserAgent->setText(model->formatSubVersion()); @@ -537,7 +631,7 @@ void RPCConsole::peerSelected(const QItemSelection &selected, const QItemSelecti { Q_UNUSED(deselected); - if (!clientModel || selected.indexes().isEmpty()) + if (!clientModel || !clientModel->getPeerTableModel() || selected.indexes().isEmpty()) return; const CNodeCombinedStats *stats = clientModel->getPeerTableModel()->getNodeStats(selected.indexes().first().row()); @@ -547,7 +641,7 @@ void RPCConsole::peerSelected(const QItemSelection &selected, const QItemSelecti void RPCConsole::peerLayoutChanged() { - if (!clientModel) + if (!clientModel || !clientModel->getPeerTableModel()) return; const CNodeCombinedStats *stats = NULL; @@ -656,7 +750,7 @@ void RPCConsole::showEvent(QShowEvent *event) { QWidget::showEvent(event); - if (!clientModel) + if (!clientModel || !clientModel->getPeerTableModel()) return; // start PeerTableModel auto refresh @@ -667,18 +761,25 @@ void RPCConsole::hideEvent(QHideEvent *event) { QWidget::hideEvent(event); - if (!clientModel) + if (!clientModel || !clientModel->getPeerTableModel()) return; // stop PeerTableModel auto refresh clientModel->getPeerTableModel()->stopAutoRefresh(); } -void RPCConsole::showMenu(const QPoint& point) +void RPCConsole::showPeersTableContextMenu(const QPoint& point) { QModelIndex index = ui->peerWidget->indexAt(point); if (index.isValid()) - contextMenu->exec(QCursor::pos()); + peersTableContextMenu->exec(QCursor::pos()); +} + +void RPCConsole::showBanTableContextMenu(const QPoint& point) +{ + QModelIndex index = ui->banlistWidget->indexAt(point); + if (index.isValid()) + banTableContextMenu->exec(QCursor::pos()); } void RPCConsole::disconnectSelectedNode() @@ -692,6 +793,46 @@ void RPCConsole::disconnectSelectedNode() } } +void RPCConsole::banSelectedNode(int bantime) +{ + if (!clientModel) + return; + + // Get currently selected peer address + QString strNode = GUIUtil::getEntryData(ui->peerWidget, 0, PeerTableModel::Address); + // Find possible nodes, ban it and clear the selected node + if (CNode *bannedNode = FindNode(strNode.toStdString())) { + std::string nStr = strNode.toStdString(); + std::string addr; + int port = 0; + SplitHostPort(nStr, port, addr); + + CNode::Ban(CNetAddr(addr), BanReasonManuallyAdded, bantime); + bannedNode->fDisconnect = true; + DumpBanlist(); + + clearSelectedNode(); + clientModel->getBanTableModel()->refresh(); + } +} + +void RPCConsole::unbanSelectedNode() +{ + if (!clientModel) + return; + + // Get currently selected ban address + QString strNode = GUIUtil::getEntryData(ui->banlistWidget, 0, BanTableModel::Address); + CSubNet possibleSubnet(strNode.toStdString()); + + if (possibleSubnet.IsValid()) + { + CNode::Unban(possibleSubnet); + DumpBanlist(); + clientModel->getBanTableModel()->refresh(); + } +} + void RPCConsole::clearSelectedNode() { ui->peerWidget->selectionModel()->clearSelection(); @@ -699,3 +840,13 @@ void RPCConsole::clearSelectedNode() ui->detailWidget->hide(); ui->peerHeading->setText(tr("Select a peer to view detailed information.")); } + +void RPCConsole::showOrHideBanTableIfRequired() +{ + if (!clientModel) + return; + + bool visible = clientModel->getBanTableModel()->shouldShow(); + ui->banlistWidget->setVisible(visible); + ui->banHeading->setVisible(visible); +}
\ No newline at end of file diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index b94efee84a..b86f776786 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -14,6 +14,7 @@ class ClientModel; class PlatformStyle; +class RPCTimerInterface; namespace Ui { class RPCConsole; @@ -60,7 +61,13 @@ private Q_SLOTS: void showEvent(QShowEvent *event); void hideEvent(QHideEvent *event); /** Show custom context menu on Peers tab */ - void showMenu(const QPoint& point); + void showPeersTableContextMenu(const QPoint& point); + /** Show custom context menu on Bans tab */ + void showBanTableContextMenu(const QPoint& point); + /** Hides ban table if no bans are present */ + void showOrHideBanTableIfRequired(); + /** clear the selected node */ + void clearSelectedNode(); public Q_SLOTS: void clear(); @@ -79,6 +86,10 @@ public Q_SLOTS: void peerLayoutChanged(); /** Disconnect a selected node on the Peers tab */ void disconnectSelectedNode(); + /** Ban a selected node on the Peers tab */ + void banSelectedNode(int bantime); + /** Unban a selected node on the Bans tab */ + void unbanSelectedNode(); Q_SIGNALS: // For RPC command executor @@ -91,14 +102,15 @@ private: void setTrafficGraphRange(int mins); /** show detailed information on ui about selected node */ void updateNodeDetail(const CNodeCombinedStats *stats); - /** clear the selected node */ - void clearSelectedNode(); enum ColumnWidths { ADDRESS_COLUMN_WIDTH = 200, SUBVERSION_COLUMN_WIDTH = 100, - PING_COLUMN_WIDTH = 80 + PING_COLUMN_WIDTH = 80, + BANSUBNET_COLUMN_WIDTH = 200, + BANTIME_COLUMN_WIDTH = 250 + }; Ui::RPCConsole *ui; @@ -106,8 +118,10 @@ private: QStringList history; int historyPtr; NodeId cachedNodeid; - QMenu *contextMenu; const PlatformStyle *platformStyle; + RPCTimerInterface *rpcTimerInterface; + QMenu *peersTableContextMenu; + QMenu *banTableContextMenu; }; #endif // BITCOIN_QT_RPCCONSOLE_H diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 60a3fc128e..a083a6f80e 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -312,9 +312,9 @@ void SendCoinsDialog::on_sendButton_clicked() if(u != model->getOptionsModel()->getDisplayUnit()) alternativeUnits.append(BitcoinUnits::formatHtmlWithUnit(u, totalAmount)); } - questionString.append(tr("Total Amount %1 (= %2)") + questionString.append(tr("Total Amount %1<span style='font-size:10pt;font-weight:normal;'><br />(=%2)</span>") .arg(BitcoinUnits::formatHtmlWithUnit(model->getOptionsModel()->getDisplayUnit(), totalAmount)) - .arg(alternativeUnits.join(" " + tr("or") + " "))); + .arg(alternativeUnits.join(" " + tr("or") + "<br />"))); QMessageBox::StandardButton retval = QMessageBox::question(this, tr("Confirm send coins"), questionString.arg(formatted.join("<br />")), diff --git a/src/qt/test/paymentservertests.cpp b/src/qt/test/paymentservertests.cpp index b28934cd31..fa5696325d 100644 --- a/src/qt/test/paymentservertests.cpp +++ b/src/qt/test/paymentservertests.cpp @@ -185,7 +185,8 @@ void PaymentServerTests::paymentServerTests() tempFile.open(); tempFile.write((const char*)randData, sizeof(randData)); tempFile.close(); - QCOMPARE(PaymentServer::readPaymentRequestFromFile(tempFile.fileName(), r.paymentRequest), false); + // compares 50001 <= BIP70_MAX_PAYMENTREQUEST_SIZE == false + QCOMPARE(PaymentServer::verifySize(tempFile.size()), false); // Payment request with amount overflow (amount is set to 21000001 BTC): data = DecodeBase64(paymentrequest5_cert2_BASE64); diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index bb768f1325..f91de2008c 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -17,6 +17,8 @@ #include <QObject> #include <QTest> +#include <openssl/ssl.h> + #if defined(QT_STATICPLUGIN) && QT_VERSION < 0x050000 #include <QtPlugin> Q_IMPORT_PLUGIN(qcncodecs) @@ -36,6 +38,8 @@ int main(int argc, char *argv[]) QCoreApplication app(argc, argv); app.setApplicationName("Bitcoin-Qt-test"); + SSL_library_init(); + URITests test1; if (QTest::qExec(&test1) != 0) fInvalid = true; diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index fa96f62e03..77efdb5cdd 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -56,6 +56,9 @@ WalletView::WalletView(const PlatformStyle *platformStyle, QWidget *parent): receiveCoinsPage = new ReceiveCoinsDialog(platformStyle); sendCoinsPage = new SendCoinsDialog(platformStyle); + usedSendingAddressesPage = new AddressBookPage(platformStyle, AddressBookPage::ForEditing, AddressBookPage::SendingTab, this); + usedReceivingAddressesPage = new AddressBookPage(platformStyle, AddressBookPage::ForEditing, AddressBookPage::ReceivingTab, this); + addWidget(overviewPage); addWidget(transactionsPage); addWidget(receiveCoinsPage); @@ -115,6 +118,8 @@ void WalletView::setWalletModel(WalletModel *walletModel) overviewPage->setWalletModel(walletModel); receiveCoinsPage->setModel(walletModel); sendCoinsPage->setModel(walletModel); + usedReceivingAddressesPage->setModel(walletModel->getAddressTableModel()); + usedSendingAddressesPage->setModel(walletModel->getAddressTableModel()); if (walletModel) { @@ -273,20 +278,20 @@ void WalletView::usedSendingAddresses() { if(!walletModel) return; - AddressBookPage *dlg = new AddressBookPage(platformStyle, AddressBookPage::ForEditing, AddressBookPage::SendingTab, this); - dlg->setAttribute(Qt::WA_DeleteOnClose); - dlg->setModel(walletModel->getAddressTableModel()); - dlg->show(); + + usedSendingAddressesPage->show(); + usedSendingAddressesPage->raise(); + usedSendingAddressesPage->activateWindow(); } void WalletView::usedReceivingAddresses() { if(!walletModel) return; - AddressBookPage *dlg = new AddressBookPage(platformStyle, AddressBookPage::ForEditing, AddressBookPage::ReceivingTab, this); - dlg->setAttribute(Qt::WA_DeleteOnClose); - dlg->setModel(walletModel->getAddressTableModel()); - dlg->show(); + + usedReceivingAddressesPage->show(); + usedReceivingAddressesPage->raise(); + usedReceivingAddressesPage->activateWindow(); } void WalletView::showProgress(const QString &title, int nProgress) diff --git a/src/qt/walletview.h b/src/qt/walletview.h index f97cf1ee80..2a6a6a2df2 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -18,6 +18,7 @@ class SendCoinsDialog; class SendCoinsRecipient; class TransactionView; class WalletModel; +class AddressBookPage; QT_BEGIN_NAMESPACE class QModelIndex; @@ -61,6 +62,8 @@ private: QWidget *transactionsPage; ReceiveCoinsDialog *receiveCoinsPage; SendCoinsDialog *sendCoinsPage; + AddressBookPage *usedSendingAddressesPage; + AddressBookPage *usedReceivingAddressesPage; TransactionView *transactionView; diff --git a/src/rest.cpp b/src/rest.cpp index 74d27e73bb..226e237fc6 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -7,6 +7,7 @@ #include "primitives/block.h" #include "primitives/transaction.h" #include "main.h" +#include "httpserver.h" #include "rpcserver.h" #include "streams.h" #include "sync.h" @@ -56,13 +57,6 @@ struct CCoin { } }; -class RestErr -{ -public: - enum HTTPStatusCode status; - string message; -}; - extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); extern UniValue mempoolInfoToJSON(); @@ -70,23 +64,31 @@ extern UniValue mempoolToJSON(bool fVerbose = false); extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); extern UniValue blockheaderToJSON(const CBlockIndex* blockindex); -static RestErr RESTERR(enum HTTPStatusCode status, string message) +static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, string message) { - RestErr re; - re.status = status; - re.message = message; - return re; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(status, message + "\r\n"); + return false; } -static enum RetFormat ParseDataFormat(vector<string>& params, const string strReq) +static enum RetFormat ParseDataFormat(std::string& param, const std::string& strReq) { - boost::split(params, strReq, boost::is_any_of(".")); - if (params.size() > 1) { - for (unsigned int i = 0; i < ARRAYLEN(rf_names); i++) - if (params[1] == rf_names[i].name) - return rf_names[i].rf; + const std::string::size_type pos = strReq.rfind('.'); + if (pos == std::string::npos) + { + param = strReq; + return rf_names[0].rf; } + param = strReq.substr(0, pos); + const std::string suff(strReq, pos + 1); + + for (unsigned int i = 0; i < ARRAYLEN(rf_names); i++) + if (suff == rf_names[i].name) + return rf_names[i].rf; + + /* If no suffix is found, return original string. */ + param = strReq; return rf_names[0].rf; } @@ -115,28 +117,35 @@ static bool ParseHashStr(const string& strReq, uint256& v) return true; } -static bool rest_headers(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) +static bool CheckWarmup(HTTPRequest* req) +{ + std::string statusmessage; + if (RPCIsInWarmup(&statusmessage)) + return RESTERR(req, HTTP_SERVICE_UNAVAILABLE, "Service temporarily unavailable: " + statusmessage); + return true; +} + +static bool rest_headers(HTTPRequest* req, + const std::string& strURIPart) { - vector<string> params; - const RetFormat rf = ParseDataFormat(params, strURIPart); + if (!CheckWarmup(req)) + return false; + std::string param; + const RetFormat rf = ParseDataFormat(param, strURIPart); vector<string> path; - boost::split(path, params[0], boost::is_any_of("/")); + boost::split(path, param, boost::is_any_of("/")); if (path.size() != 2) - throw RESTERR(HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>."); + return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>."); long count = strtol(path[0].c_str(), NULL, 10); if (count < 1 || count > 2000) - throw RESTERR(HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); + return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); string hashStr = path[1]; uint256 hash; if (!ParseHashStr(hashStr, hash)) - throw RESTERR(HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); std::vector<const CBlockIndex *> headers; headers.reserve(count); @@ -160,28 +169,29 @@ static bool rest_headers(AcceptedConnection* conn, switch (rf) { case RF_BINARY: { string binaryHeader = ssHeader.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, binaryHeader.size(), "application/octet-stream") << binaryHeader << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryHeader); return true; } case RF_HEX: { string strHex = HexStr(ssHeader.begin(), ssHeader.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } - case RF_JSON: { UniValue jsonHeaders(UniValue::VARR); BOOST_FOREACH(const CBlockIndex *pindex, headers) { jsonHeaders.push_back(blockheaderToJSON(pindex)); } string strJSON = jsonHeaders.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } - default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: .bin, .hex)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex)"); } } @@ -189,34 +199,32 @@ static bool rest_headers(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_block(AcceptedConnection* conn, +static bool rest_block(HTTPRequest* req, const std::string& strURIPart, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun, bool showTxDetails) { - vector<string> params; - const RetFormat rf = ParseDataFormat(params, strURIPart); + if (!CheckWarmup(req)) + return false; + std::string hashStr; + const RetFormat rf = ParseDataFormat(hashStr, strURIPart); - string hashStr = params[0]; uint256 hash; if (!ParseHashStr(hashStr, hash)) - throw RESTERR(HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); CBlock block; CBlockIndex* pblockindex = NULL; { LOCK(cs_main); if (mapBlockIndex.count(hash) == 0) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); pblockindex = mapBlockIndex[hash]; if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); if (!ReadBlockFromDisk(block, pblockindex)) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); @@ -225,25 +233,28 @@ static bool rest_block(AcceptedConnection* conn, switch (rf) { case RF_BINARY: { string binaryBlock = ssBlock.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, binaryBlock.size(), "application/octet-stream") << binaryBlock << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryBlock); return true; } case RF_HEX: { string strHex = HexStr(ssBlock.begin(), ssBlock.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } case RF_JSON: { UniValue objBlock = blockToJSON(block, pblockindex, showTxDetails); string strJSON = objBlock.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } @@ -251,43 +262,34 @@ static bool rest_block(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_block_extended(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) +static bool rest_block_extended(HTTPRequest* req, const std::string& strURIPart) { - return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, true); + return rest_block(req, strURIPart, true); } -static bool rest_block_notxdetails(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) +static bool rest_block_notxdetails(HTTPRequest* req, const std::string& strURIPart) { - return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, false); + return rest_block(req, strURIPart, false); } -static bool rest_chaininfo(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) +static bool rest_chaininfo(HTTPRequest* req, const std::string& strURIPart) { - vector<string> params; - const RetFormat rf = ParseDataFormat(params, strURIPart); + if (!CheckWarmup(req)) + return false; + std::string param; + const RetFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { case RF_JSON: { UniValue rpcParams(UniValue::VARR); UniValue chainInfoObject = getblockchaininfo(rpcParams, false); string strJSON = chainInfoObject.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } @@ -295,25 +297,24 @@ static bool rest_chaininfo(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_mempool_info(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) +static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) { - vector<string> params; - const RetFormat rf = ParseDataFormat(params, strURIPart); + if (!CheckWarmup(req)) + return false; + std::string param; + const RetFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { case RF_JSON: { UniValue mempoolInfoObject = mempoolInfoToJSON(); string strJSON = mempoolInfoObject.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } @@ -321,25 +322,24 @@ static bool rest_mempool_info(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_mempool_contents(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) +static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPart) { - vector<string> params; - const RetFormat rf = ParseDataFormat(params, strURIPart); + if (!CheckWarmup(req)) + return false; + std::string param; + const RetFormat rf = ParseDataFormat(param, strURIPart); switch (rf) { case RF_JSON: { UniValue mempoolObject = mempoolToJSON(true); string strJSON = mempoolObject.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } @@ -347,24 +347,21 @@ static bool rest_mempool_contents(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_tx(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) +static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) { - vector<string> params; - const RetFormat rf = ParseDataFormat(params, strURIPart); + if (!CheckWarmup(req)) + return false; + std::string hashStr; + const RetFormat rf = ParseDataFormat(hashStr, strURIPart); - string hashStr = params[0]; uint256 hash; if (!ParseHashStr(hashStr, hash)) - throw RESTERR(HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); CTransaction tx; uint256 hashBlock = uint256(); if (!GetTransaction(hash, tx, hashBlock, true)) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << tx; @@ -372,13 +369,15 @@ static bool rest_tx(AcceptedConnection* conn, switch (rf) { case RF_BINARY: { string binaryTx = ssTx.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, binaryTx.size(), "application/octet-stream") << binaryTx << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryTx); return true; } case RF_HEX: { string strHex = HexStr(ssTx.begin(), ssTx.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } @@ -386,12 +385,13 @@ static bool rest_tx(AcceptedConnection* conn, UniValue objTx(UniValue::VOBJ); TxToJSON(tx, hashBlock, objTx); string strJSON = objTx.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } @@ -399,25 +399,24 @@ static bool rest_tx(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_getutxos(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) +static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) { - vector<string> params; - enum RetFormat rf = ParseDataFormat(params, strURIPart); + if (!CheckWarmup(req)) + return false; + std::string param; + const RetFormat rf = ParseDataFormat(param, strURIPart); vector<string> uriParts; - if (params.size() > 0 && params[0].length() > 1) + if (param.length() > 1) { - std::string strUriParams = params[0].substr(1); + std::string strUriParams = param.substr(1); boost::split(uriParts, strUriParams, boost::is_any_of("/")); } // throw exception in case of a empty request - if (strRequest.length() == 0 && uriParts.size() == 0) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + std::string strRequestMutable = req->ReadBody(); + if (strRequestMutable.length() == 0 && uriParts.size() == 0) + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); bool fInputParsed = false; bool fCheckMemPool = false; @@ -441,7 +440,7 @@ static bool rest_getutxos(AcceptedConnection* conn, std::string strOutput = uriParts[i].substr(uriParts[i].find("-")+1); if (!ParseInt32(strOutput, &nOutput) || !IsHex(strTxid)) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Parse error"); txid.SetHex(strTxid); vOutPoints.push_back(COutPoint(txid, (uint32_t)nOutput)); @@ -450,15 +449,13 @@ static bool rest_getutxos(AcceptedConnection* conn, if (vOutPoints.size() > 0) fInputParsed = true; else - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); } - string strRequestMutable = strRequest; //convert const string to string for allowing hex to bin converting - switch (rf) { case RF_HEX: { // convert hex to bin, continue then with bin part - std::vector<unsigned char> strRequestV = ParseHex(strRequest); + std::vector<unsigned char> strRequestV = ParseHex(strRequestMutable); strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); } @@ -468,7 +465,7 @@ static bool rest_getutxos(AcceptedConnection* conn, if (strRequestMutable.size() > 0) { if (fInputParsed) //don't allow sending input over URI and HTTP RAW DATA - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Combination of URI scheme inputs and raw post data is not allowed"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Combination of URI scheme inputs and raw post data is not allowed"); CDataStream oss(SER_NETWORK, PROTOCOL_VERSION); oss << strRequestMutable; @@ -477,24 +474,24 @@ static bool rest_getutxos(AcceptedConnection* conn, } } catch (const std::ios_base::failure& e) { // abort in case of unreadable binary data - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Parse error"); } break; } case RF_JSON: { if (!fInputParsed) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); break; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } // limit max outpoints if (vOutPoints.size() > MAX_GETUTXOS_OUTPOINTS) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, strprintf("Error: max outpoints exceeded (max: %d, tried: %d)", MAX_GETUTXOS_OUTPOINTS, vOutPoints.size())); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, strprintf("Error: max outpoints exceeded (max: %d, tried: %d)", MAX_GETUTXOS_OUTPOINTS, vOutPoints.size())); // check spentness and form a bitmap (as well as a JSON capable human-readble string representation) vector<unsigned char> bitmap; @@ -544,7 +541,8 @@ static bool rest_getutxos(AcceptedConnection* conn, ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; string ssGetUTXOResponseString = ssGetUTXOResponse.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, ssGetUTXOResponseString.size(), "application/octet-stream") << ssGetUTXOResponseString << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, ssGetUTXOResponseString); return true; } @@ -553,7 +551,8 @@ static bool rest_getutxos(AcceptedConnection* conn, ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; string strHex = HexStr(ssGetUTXOResponse.begin(), ssGetUTXOResponse.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } @@ -583,11 +582,12 @@ static bool rest_getutxos(AcceptedConnection* conn, // return json string string strJSON = objGetUTXOResponse.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } @@ -597,11 +597,7 @@ static bool rest_getutxos(AcceptedConnection* conn, static const struct { const char* prefix; - bool (*handler)(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun); + bool (*handler)(HTTPRequest* req, const std::string& strReq); } uri_prefixes[] = { {"/rest/tx/", rest_tx}, {"/rest/block/notxdetails/", rest_block_notxdetails}, @@ -613,29 +609,19 @@ static const struct { {"/rest/getutxos", rest_getutxos}, }; -bool HTTPReq_REST(AcceptedConnection* conn, - const std::string& strURI, - const string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun) +bool StartREST() { - try { - std::string statusmessage; - if (RPCIsInWarmup(&statusmessage)) - throw RESTERR(HTTP_SERVICE_UNAVAILABLE, "Service temporarily unavailable: " + statusmessage); - - for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) { - unsigned int plen = strlen(uri_prefixes[i].prefix); - if (strURI.substr(0, plen) == uri_prefixes[i].prefix) { - string strURIPart = strURI.substr(plen); - return uri_prefixes[i].handler(conn, strURIPart, strRequest, mapHeaders, fRun); - } - } - } catch (const RestErr& re) { - conn->stream() << HTTPReply(re.status, re.message + "\r\n", false, false, "text/plain") << std::flush; - return false; - } + for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) + RegisterHTTPHandler(uri_prefixes[i].prefix, false, uri_prefixes[i].handler); + return true; +} - conn->stream() << HTTPError(HTTP_NOT_FOUND, false) << std::flush; - return false; +void InterruptREST() +{ +} + +void StopREST() +{ + for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) + UnregisterHTTPHandler(uri_prefixes[i].prefix, false); } diff --git a/src/reverselock.h b/src/reverselock.h new file mode 100644 index 0000000000..567636e16a --- /dev/null +++ b/src/reverselock.h @@ -0,0 +1,31 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_REVERSELOCK_H +#define BITCOIN_REVERSELOCK_H + +/** + * An RAII-style reverse lock. Unlocks on construction and locks on destruction. + */ +template<typename Lock> +class reverse_lock +{ +public: + + explicit reverse_lock(Lock& lock) : lock(lock) { + lock.unlock(); + } + + ~reverse_lock() { + lock.lock(); + } + +private: + reverse_lock(reverse_lock const&); + reverse_lock& operator=(reverse_lock const&); + + Lock& lock; +}; + +#endif // BITCOIN_REVERSELOCK_H diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index e6751de96b..1c201ef99d 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -181,10 +181,9 @@ UniValue mempoolToJSON(bool fVerbose = false) { LOCK(mempool.cs); UniValue o(UniValue::VOBJ); - BOOST_FOREACH(const PAIRTYPE(uint256, CTxMemPoolEntry)& entry, mempool.mapTx) + BOOST_FOREACH(const CTxMemPoolEntry& e, mempool.mapTx) { - const uint256& hash = entry.first; - const CTxMemPoolEntry& e = entry.second; + const uint256& hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); info.push_back(Pair("size", (int)e.GetTxSize())); info.push_back(Pair("fee", ValueFromAmount(e.GetFee()))); @@ -192,6 +191,9 @@ UniValue mempoolToJSON(bool fVerbose = false) info.push_back(Pair("height", (int)e.GetHeight())); info.push_back(Pair("startingpriority", e.GetPriority(e.GetHeight()))); info.push_back(Pair("currentpriority", e.GetPriority(chainActive.Height()))); + info.push_back(Pair("descendantcount", e.GetCountWithDescendants())); + info.push_back(Pair("descendantsize", e.GetSizeWithDescendants())); + info.push_back(Pair("descendantfees", e.GetFeesWithDescendants())); const CTransaction& tx = e.GetTx(); set<string> setDepends; BOOST_FOREACH(const CTxIn& txin, tx.vin) @@ -246,6 +248,9 @@ UniValue getrawmempool(const UniValue& params, bool fHelp) " \"height\" : n, (numeric) block height when transaction entered pool\n" " \"startingpriority\" : n, (numeric) priority when transaction entered pool\n" " \"currentpriority\" : n, (numeric) transaction priority now\n" + " \"descendantcount\" : n, (numeric) number of in-mempool descendant transactions (including this one)\n" + " \"descendantsize\" : n, (numeric) size of in-mempool descendants (including this one)\n" + " \"descendantfees\" : n, (numeric) fees of in-mempool descendants (including this one)\n" " \"depends\" : [ (array) unconfirmed transactions used as inputs for this transaction\n" " \"transactionid\", (string) parent transaction id\n" " ... ]\n" diff --git a/src/rpcmining.cpp b/src/rpcmining.cpp index 620a46be15..91de37fdcc 100644 --- a/src/rpcmining.cpp +++ b/src/rpcmining.cpp @@ -119,7 +119,7 @@ UniValue generate(const UniValue& params, bool fHelp) "generate numblocks\n" "\nMine blocks immediately (before the RPC call returns)\n" "\nNote: this function can only be used on the regtest network\n" - "1. numblocks (numeric) How many blocks are generated immediately.\n" + "1. numblocks (numeric, required) How many blocks are generated immediately.\n" "\nResult\n" "[ blockhashes ] (array) hashes of blocks generated\n" "\nExamples:\n" diff --git a/src/rpcnet.cpp b/src/rpcnet.cpp index 56ec59171d..5d490c70ca 100644 --- a/src/rpcnet.cpp +++ b/src/rpcnet.cpp @@ -12,6 +12,7 @@ #include "protocol.h" #include "sync.h" #include "timedata.h" +#include "ui_interface.h" #include "util.h" #include "utilstrencodings.h" #include "version.h" @@ -96,6 +97,7 @@ UniValue getpeerinfo(const UniValue& params, bool fHelp) " \"conntime\": ttt, (numeric) The connection time in seconds since epoch (Jan 1 1970 GMT)\n" " \"timeoffset\": ttt, (numeric) The time offset in seconds\n" " \"pingtime\": n, (numeric) ping time\n" + " \"minping\": n, (numeric) minimum observed ping time\n" " \"pingwait\": n, (numeric) ping wait\n" " \"version\": v, (numeric) The peer version, such as 7001\n" " \"subver\": \"/Satoshi:0.8.5/\", (string) The string version\n" @@ -139,6 +141,7 @@ UniValue getpeerinfo(const UniValue& params, bool fHelp) obj.push_back(Pair("conntime", stats.nTimeConnected)); obj.push_back(Pair("timeoffset", stats.nTimeOffset)); obj.push_back(Pair("pingtime", stats.dPingTime)); + obj.push_back(Pair("minping", stats.dPingMin)); if (stats.dPingWait > 0.0) obj.push_back(Pair("pingwait", stats.dPingWait)); obj.push_back(Pair("version", stats.nVersion)); @@ -529,6 +532,8 @@ UniValue setban(const UniValue& params, bool fHelp) } DumpBanlist(); //store banlist to disk + uiInterface.BannedListChanged(); + return NullUniValue; } @@ -575,6 +580,7 @@ UniValue clearbanned(const UniValue& params, bool fHelp) CNode::ClearBanned(); DumpBanlist(); //store banlist to disk + uiInterface.BannedListChanged(); return NullUniValue; } diff --git a/src/rpcprotocol.cpp b/src/rpcprotocol.cpp index 2e5c913734..d83cd87f94 100644 --- a/src/rpcprotocol.cpp +++ b/src/rpcprotocol.cpp @@ -5,7 +5,6 @@ #include "rpcprotocol.h" -#include "clientversion.h" #include "random.h" #include "tinyformat.h" #include "util.h" @@ -16,236 +15,8 @@ #include <stdint.h> #include <fstream> -#include <boost/algorithm/string.hpp> -#include <boost/asio.hpp> -#include <boost/asio/ssl.hpp> -#include <boost/bind.hpp> -#include <boost/filesystem.hpp> -#include <boost/foreach.hpp> -#include <boost/iostreams/concepts.hpp> -#include <boost/iostreams/stream.hpp> -#include <boost/shared_ptr.hpp> - -#include "univalue/univalue.h" - using namespace std; -//! Number of bytes to allocate and read at most at once in post data -const size_t POST_READ_SIZE = 256 * 1024; - -/** - * HTTP protocol - * - * This ain't Apache. We're just using HTTP header for the length field - * and to be compatible with other JSON-RPC implementations. - */ - -string HTTPPost(const string& strMsg, const map<string,string>& mapRequestHeaders) -{ - ostringstream s; - s << "POST / HTTP/1.1\r\n" - << "User-Agent: bitcoin-json-rpc/" << FormatFullVersion() << "\r\n" - << "Host: 127.0.0.1\r\n" - << "Content-Type: application/json\r\n" - << "Content-Length: " << strMsg.size() << "\r\n" - << "Connection: close\r\n" - << "Accept: application/json\r\n"; - BOOST_FOREACH(const PAIRTYPE(string, string)& item, mapRequestHeaders) - s << item.first << ": " << item.second << "\r\n"; - s << "\r\n" << strMsg; - - return s.str(); -} - -static string rfc1123Time() -{ - return DateTimeStrFormat("%a, %d %b %Y %H:%M:%S +0000", GetTime()); -} - -static const char *httpStatusDescription(int nStatus) -{ - switch (nStatus) { - case HTTP_OK: return "OK"; - case HTTP_BAD_REQUEST: return "Bad Request"; - case HTTP_FORBIDDEN: return "Forbidden"; - case HTTP_NOT_FOUND: return "Not Found"; - case HTTP_INTERNAL_SERVER_ERROR: return "Internal Server Error"; - default: return ""; - } -} - -string HTTPError(int nStatus, bool keepalive, bool headersOnly) -{ - if (nStatus == HTTP_UNAUTHORIZED) - return strprintf("HTTP/1.0 401 Authorization Required\r\n" - "Date: %s\r\n" - "Server: bitcoin-json-rpc/%s\r\n" - "WWW-Authenticate: Basic realm=\"jsonrpc\"\r\n" - "Content-Type: text/html\r\n" - "Content-Length: 296\r\n" - "\r\n" - "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\"\r\n" - "\"http://www.w3.org/TR/1999/REC-html401-19991224/loose.dtd\">\r\n" - "<HTML>\r\n" - "<HEAD>\r\n" - "<TITLE>Error</TITLE>\r\n" - "<META HTTP-EQUIV='Content-Type' CONTENT='text/html; charset=ISO-8859-1'>\r\n" - "</HEAD>\r\n" - "<BODY><H1>401 Unauthorized.</H1></BODY>\r\n" - "</HTML>\r\n", rfc1123Time(), FormatFullVersion()); - - return HTTPReply(nStatus, httpStatusDescription(nStatus), keepalive, - headersOnly, "text/plain"); -} - -string HTTPReplyHeader(int nStatus, bool keepalive, size_t contentLength, const char *contentType) -{ - return strprintf( - "HTTP/1.1 %d %s\r\n" - "Date: %s\r\n" - "Connection: %s\r\n" - "Content-Length: %u\r\n" - "Content-Type: %s\r\n" - "Server: bitcoin-json-rpc/%s\r\n" - "\r\n", - nStatus, - httpStatusDescription(nStatus), - rfc1123Time(), - keepalive ? "keep-alive" : "close", - contentLength, - contentType, - FormatFullVersion()); -} - -string HTTPReply(int nStatus, const string& strMsg, bool keepalive, - bool headersOnly, const char *contentType) -{ - if (headersOnly) - { - return HTTPReplyHeader(nStatus, keepalive, 0, contentType); - } else { - return HTTPReplyHeader(nStatus, keepalive, strMsg.size(), contentType) + strMsg; - } -} - -bool ReadHTTPRequestLine(std::basic_istream<char>& stream, int &proto, - string& http_method, string& http_uri) -{ - string str; - getline(stream, str); - - // HTTP request line is space-delimited - vector<string> vWords; - boost::split(vWords, str, boost::is_any_of(" ")); - if (vWords.size() < 2) - return false; - - // HTTP methods permitted: GET, POST - http_method = vWords[0]; - if (http_method != "GET" && http_method != "POST") - return false; - - // HTTP URI must be an absolute path, relative to current host - http_uri = vWords[1]; - if (http_uri.size() == 0 || http_uri[0] != '/') - return false; - - // parse proto, if present - string strProto = ""; - if (vWords.size() > 2) - strProto = vWords[2]; - - proto = 0; - const char *ver = strstr(strProto.c_str(), "HTTP/1."); - if (ver != NULL) - proto = atoi(ver+7); - - return true; -} - -int ReadHTTPStatus(std::basic_istream<char>& stream, int &proto) -{ - string str; - getline(stream, str); - vector<string> vWords; - boost::split(vWords, str, boost::is_any_of(" ")); - if (vWords.size() < 2) - return HTTP_INTERNAL_SERVER_ERROR; - proto = 0; - const char *ver = strstr(str.c_str(), "HTTP/1."); - if (ver != NULL) - proto = atoi(ver+7); - return atoi(vWords[1].c_str()); -} - -int ReadHTTPHeaders(std::basic_istream<char>& stream, map<string, string>& mapHeadersRet) -{ - int nLen = 0; - while (true) - { - string str; - std::getline(stream, str); - if (str.empty() || str == "\r") - break; - string::size_type nColon = str.find(":"); - if (nColon != string::npos) - { - string strHeader = str.substr(0, nColon); - boost::trim(strHeader); - boost::to_lower(strHeader); - string strValue = str.substr(nColon+1); - boost::trim(strValue); - mapHeadersRet[strHeader] = strValue; - if (strHeader == "content-length") - nLen = atoi(strValue.c_str()); - } - } - return nLen; -} - - -int ReadHTTPMessage(std::basic_istream<char>& stream, map<string, - string>& mapHeadersRet, string& strMessageRet, - int nProto, size_t max_size) -{ - mapHeadersRet.clear(); - strMessageRet = ""; - - // Read header - int nLen = ReadHTTPHeaders(stream, mapHeadersRet); - if (nLen < 0 || (size_t)nLen > max_size) - return HTTP_INTERNAL_SERVER_ERROR; - - // Read message - if (nLen > 0) - { - vector<char> vch; - size_t ptr = 0; - while (ptr < (size_t)nLen) - { - size_t bytes_to_read = std::min((size_t)nLen - ptr, POST_READ_SIZE); - vch.resize(ptr + bytes_to_read); - stream.read(&vch[ptr], bytes_to_read); - if (!stream) // Connection lost while reading - return HTTP_INTERNAL_SERVER_ERROR; - ptr += bytes_to_read; - } - strMessageRet = string(vch.begin(), vch.end()); - } - - string sConHdr = mapHeadersRet["connection"]; - - if ((sConHdr != "close") && (sConHdr != "keep-alive")) - { - if (nProto >= 1) - mapHeadersRet["connection"] = "keep-alive"; - else - mapHeadersRet["connection"] = "close"; - } - - return HTTP_OK; -} - /** * JSON-RPC protocol. Bitcoin speaks version 1.0 for maximum compatibility, * but uses JSON-RPC 1.1/2.0 standards for parts of the 1.0 standard that were diff --git a/src/rpcprotocol.h b/src/rpcprotocol.h index 2360ec2c60..5381e4bcfd 100644 --- a/src/rpcprotocol.h +++ b/src/rpcprotocol.h @@ -10,10 +10,6 @@ #include <map> #include <stdint.h> #include <string> -#include <boost/iostreams/concepts.hpp> -#include <boost/iostreams/stream.hpp> -#include <boost/asio.hpp> -#include <boost/asio/ssl.hpp> #include <boost/filesystem.hpp> #include "univalue/univalue.h" @@ -26,6 +22,7 @@ enum HTTPStatusCode HTTP_UNAUTHORIZED = 401, HTTP_FORBIDDEN = 403, HTTP_NOT_FOUND = 404, + HTTP_BAD_METHOD = 405, HTTP_INTERNAL_SERVER_ERROR = 500, HTTP_SERVICE_UNAVAILABLE = 503, }; @@ -79,88 +76,6 @@ enum RPCErrorCode RPC_WALLET_ALREADY_UNLOCKED = -17, //! Wallet is already unlocked }; -/** - * IOStream device that speaks SSL but can also speak non-SSL - */ -template <typename Protocol> -class SSLIOStreamDevice : public boost::iostreams::device<boost::iostreams::bidirectional> { -public: - SSLIOStreamDevice(boost::asio::ssl::stream<typename Protocol::socket> &streamIn, bool fUseSSLIn) : stream(streamIn) - { - fUseSSL = fUseSSLIn; - fNeedHandshake = fUseSSLIn; - } - - void handshake(boost::asio::ssl::stream_base::handshake_type role) - { - if (!fNeedHandshake) return; - fNeedHandshake = false; - stream.handshake(role); - } - std::streamsize read(char* s, std::streamsize n) - { - handshake(boost::asio::ssl::stream_base::server); // HTTPS servers read first - if (fUseSSL) return stream.read_some(boost::asio::buffer(s, n)); - return stream.next_layer().read_some(boost::asio::buffer(s, n)); - } - std::streamsize write(const char* s, std::streamsize n) - { - handshake(boost::asio::ssl::stream_base::client); // HTTPS clients write first - if (fUseSSL) return boost::asio::write(stream, boost::asio::buffer(s, n)); - return boost::asio::write(stream.next_layer(), boost::asio::buffer(s, n)); - } - bool connect(const std::string& server, const std::string& port) - { - using namespace boost::asio::ip; - tcp::resolver resolver(stream.get_io_service()); - tcp::resolver::iterator endpoint_iterator; -#if BOOST_VERSION >= 104300 - try { -#endif - // The default query (flags address_configured) tries IPv6 if - // non-localhost IPv6 configured, and IPv4 if non-localhost IPv4 - // configured. - tcp::resolver::query query(server.c_str(), port.c_str()); - endpoint_iterator = resolver.resolve(query); -#if BOOST_VERSION >= 104300 - } catch (const boost::system::system_error&) { - // If we at first don't succeed, try blanket lookup (IPv4+IPv6 independent of configured interfaces) - tcp::resolver::query query(server.c_str(), port.c_str(), resolver_query_base::flags()); - endpoint_iterator = resolver.resolve(query); - } -#endif - boost::system::error_code error = boost::asio::error::host_not_found; - tcp::resolver::iterator end; - while (error && endpoint_iterator != end) - { - stream.lowest_layer().close(); - stream.lowest_layer().connect(*endpoint_iterator++, error); - } - if (error) - return false; - return true; - } - -private: - bool fNeedHandshake; - bool fUseSSL; - boost::asio::ssl::stream<typename Protocol::socket>& stream; -}; - -std::string HTTPPost(const std::string& strMsg, const std::map<std::string,std::string>& mapRequestHeaders); -std::string HTTPError(int nStatus, bool keepalive, - bool headerOnly = false); -std::string HTTPReplyHeader(int nStatus, bool keepalive, size_t contentLength, - const char *contentType = "application/json"); -std::string HTTPReply(int nStatus, const std::string& strMsg, bool keepalive, - bool headerOnly = false, - const char *contentType = "application/json"); -bool ReadHTTPRequestLine(std::basic_istream<char>& stream, int &proto, - std::string& http_method, std::string& http_uri); -int ReadHTTPStatus(std::basic_istream<char>& stream, int &proto); -int ReadHTTPHeaders(std::basic_istream<char>& stream, std::map<std::string, std::string>& mapHeadersRet); -int ReadHTTPMessage(std::basic_istream<char>& stream, std::map<std::string, std::string>& mapHeadersRet, - std::string& strMessageRet, int nProto, size_t max_size); std::string JSONRPCRequest(const std::string& strMethod, const UniValue& params, const UniValue& id); UniValue JSONRPCReplyObj(const UniValue& result, const UniValue& error, const UniValue& id); std::string JSONRPCReply(const UniValue& result, const UniValue& error, const UniValue& id); diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 9362401b1e..b831d3d3b2 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -12,13 +12,9 @@ #include "ui_interface.h" #include "util.h" #include "utilstrencodings.h" -#ifdef ENABLE_WALLET -#include "wallet/wallet.h" -#endif -#include <boost/algorithm/string.hpp> -#include <boost/asio.hpp> -#include <boost/asio/ssl.hpp> +#include "univalue/univalue.h" + #include <boost/bind.hpp> #include <boost/filesystem.hpp> #include <boost/foreach.hpp> @@ -27,28 +23,20 @@ #include <boost/shared_ptr.hpp> #include <boost/signals2/signal.hpp> #include <boost/thread.hpp> +#include <boost/algorithm/string/case_conv.hpp> // for to_upper() -#include "univalue/univalue.h" - -using namespace boost::asio; using namespace RPCServer; using namespace std; -static std::string strRPCUserColonPass; - static bool fRPCRunning = false; static bool fRPCInWarmup = true; static std::string rpcWarmupStatus("RPC server started"); static CCriticalSection cs_rpcWarmup; - -//! These are created by StartRPCThreads, destroyed in StopRPCThreads -static boost::asio::io_service* rpc_io_service = NULL; -static map<string, boost::shared_ptr<deadline_timer> > deadlineTimers; -static ssl::context* rpc_ssl_context = NULL; -static boost::thread_group* rpc_worker_group = NULL; -static boost::asio::io_service::work *rpc_dummy_work = NULL; -static std::vector<CSubNet> rpc_allow_subnets; //!< List of subnets to allow RPC connections from -static std::vector< boost::shared_ptr<ip::tcp::acceptor> > rpc_acceptors; +/* Timer-creating functions */ +static std::vector<RPCTimerInterface*> timerInterfaces; +/* Map of name to timer. + * @note Can be changed to std::unique_ptr when C++11 */ +static std::map<std::string, boost::shared_ptr<RPCTimerBase> > deadlineTimers; static struct CRPCSignals { @@ -169,7 +157,6 @@ vector<unsigned char> ParseHexO(const UniValue& o, string strKey) return ParseHexV(find_value(o, strKey), strKey); } - /** * Note: This interface may still be subject to change. */ @@ -261,8 +248,6 @@ UniValue stop(const UniValue& params, bool fHelp) return "Bitcoin server stopping"; } - - /** * Call Table */ @@ -399,7 +384,7 @@ CRPCTable::CRPCTable() } } -const CRPCCommand *CRPCTable::operator[](const std::string& name) const +const CRPCCommand *CRPCTable::operator[](const std::string &name) const { map<string, const CRPCCommand*>::const_iterator it = mapCommands.find(name); if (it == mapCommands.end()) @@ -407,373 +392,26 @@ const CRPCCommand *CRPCTable::operator[](const std::string& name) const return (*it).second; } - -bool HTTPAuthorized(map<string, string>& mapHeaders) -{ - string strAuth = mapHeaders["authorization"]; - if (strAuth.substr(0,6) != "Basic ") - return false; - string strUserPass64 = strAuth.substr(6); boost::trim(strUserPass64); - string strUserPass = DecodeBase64(strUserPass64); - return TimingResistantEqual(strUserPass, strRPCUserColonPass); -} - -void ErrorReply(std::ostream& stream, const UniValue& objError, const UniValue& id) -{ - // Send error reply from json-rpc error object - int nStatus = HTTP_INTERNAL_SERVER_ERROR; - int code = find_value(objError, "code").get_int(); - if (code == RPC_INVALID_REQUEST) nStatus = HTTP_BAD_REQUEST; - else if (code == RPC_METHOD_NOT_FOUND) nStatus = HTTP_NOT_FOUND; - string strReply = JSONRPCReply(NullUniValue, objError, id); - stream << HTTPReply(nStatus, strReply, false) << std::flush; -} - -CNetAddr BoostAsioToCNetAddr(boost::asio::ip::address address) -{ - CNetAddr netaddr; - // Make sure that IPv4-compatible and IPv4-mapped IPv6 addresses are treated as IPv4 addresses - if (address.is_v6() - && (address.to_v6().is_v4_compatible() - || address.to_v6().is_v4_mapped())) - address = address.to_v6().to_v4(); - - if(address.is_v4()) - { - boost::asio::ip::address_v4::bytes_type bytes = address.to_v4().to_bytes(); - netaddr.SetRaw(NET_IPV4, &bytes[0]); - } - else - { - boost::asio::ip::address_v6::bytes_type bytes = address.to_v6().to_bytes(); - netaddr.SetRaw(NET_IPV6, &bytes[0]); - } - return netaddr; -} - -bool ClientAllowed(const boost::asio::ip::address& address) -{ - CNetAddr netaddr = BoostAsioToCNetAddr(address); - BOOST_FOREACH(const CSubNet &subnet, rpc_allow_subnets) - if (subnet.Match(netaddr)) - return true; - return false; -} - -template <typename Protocol> -class AcceptedConnectionImpl : public AcceptedConnection -{ -public: - AcceptedConnectionImpl( - boost::asio::io_service& io_service, - ssl::context &context, - bool fUseSSL) : - sslStream(io_service, context), - _d(sslStream, fUseSSL), - _stream(_d) - { - } - - virtual std::iostream& stream() - { - return _stream; - } - - virtual std::string peer_address_to_string() const - { - return peer.address().to_string(); - } - - virtual void close() - { - _stream.close(); - } - - typename Protocol::endpoint peer; - boost::asio::ssl::stream<typename Protocol::socket> sslStream; - -private: - SSLIOStreamDevice<Protocol> _d; - boost::iostreams::stream< SSLIOStreamDevice<Protocol> > _stream; -}; - -void ServiceConnection(AcceptedConnection *conn); - -//! Forward declaration required for RPCListen -template <typename Protocol, typename SocketAcceptorService> -static void RPCAcceptHandler(boost::shared_ptr< basic_socket_acceptor<Protocol, SocketAcceptorService> > acceptor, - ssl::context& context, - bool fUseSSL, - boost::shared_ptr< AcceptedConnection > conn, - const boost::system::error_code& error); - -/** - * Sets up I/O resources to accept and handle a new connection. - */ -template <typename Protocol, typename SocketAcceptorService> -static void RPCListen(boost::shared_ptr< basic_socket_acceptor<Protocol, SocketAcceptorService> > acceptor, - ssl::context& context, - const bool fUseSSL) -{ - // Accept connection - boost::shared_ptr< AcceptedConnectionImpl<Protocol> > conn(new AcceptedConnectionImpl<Protocol>(acceptor->get_io_service(), context, fUseSSL)); - - acceptor->async_accept( - conn->sslStream.lowest_layer(), - conn->peer, - boost::bind(&RPCAcceptHandler<Protocol, SocketAcceptorService>, - acceptor, - boost::ref(context), - fUseSSL, - conn, - _1)); -} - - -/** - * Accept and handle incoming connection. - */ -template <typename Protocol, typename SocketAcceptorService> -static void RPCAcceptHandler(boost::shared_ptr< basic_socket_acceptor<Protocol, SocketAcceptorService> > acceptor, - ssl::context& context, - const bool fUseSSL, - boost::shared_ptr< AcceptedConnection > conn, - const boost::system::error_code& error) +bool StartRPC() { - // Immediately start accepting new connections, except when we're cancelled or our socket is closed. - if (error != boost::asio::error::operation_aborted && acceptor->is_open()) - RPCListen(acceptor, context, fUseSSL); - - AcceptedConnectionImpl<ip::tcp>* tcp_conn = dynamic_cast< AcceptedConnectionImpl<ip::tcp>* >(conn.get()); - - if (error) - { - // TODO: Actually handle errors - LogPrintf("%s: Error: %s\n", __func__, error.message()); - } - // Restrict callers by IP. It is important to - // do this before starting client thread, to filter out - // certain DoS and misbehaving clients. - else if (tcp_conn && !ClientAllowed(tcp_conn->peer.address())) - { - // Only send a 403 if we're not using SSL to prevent a DoS during the SSL handshake. - if (!fUseSSL) - conn->stream() << HTTPError(HTTP_FORBIDDEN, false) << std::flush; - conn->close(); - } - else { - ServiceConnection(conn.get()); - conn->close(); - } -} - -static ip::tcp::endpoint ParseEndpoint(const std::string &strEndpoint, int defaultPort) -{ - std::string addr; - int port = defaultPort; - SplitHostPort(strEndpoint, port, addr); - return ip::tcp::endpoint(boost::asio::ip::address::from_string(addr), port); -} - -void StartRPCThreads() -{ - rpc_allow_subnets.clear(); - rpc_allow_subnets.push_back(CSubNet("127.0.0.0/8")); // always allow IPv4 local subnet - rpc_allow_subnets.push_back(CSubNet("::1")); // always allow IPv6 localhost - if (mapMultiArgs.count("-rpcallowip")) - { - const vector<string>& vAllow = mapMultiArgs["-rpcallowip"]; - BOOST_FOREACH(string strAllow, vAllow) - { - CSubNet subnet(strAllow); - if(!subnet.IsValid()) - { - uiInterface.ThreadSafeMessageBox( - strprintf("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24).", strAllow), - "", CClientUIInterface::MSG_ERROR); - StartShutdown(); - return; - } - rpc_allow_subnets.push_back(subnet); - } - } - std::string strAllowed; - BOOST_FOREACH(const CSubNet &subnet, rpc_allow_subnets) - strAllowed += subnet.ToString() + " "; - LogPrint("rpc", "Allowing RPC connections from: %s\n", strAllowed); - - if (mapArgs["-rpcpassword"] == "") - { - LogPrintf("No rpcpassword set - using random cookie authentication\n"); - if (!GenerateAuthCookie(&strRPCUserColonPass)) { - uiInterface.ThreadSafeMessageBox( - _("Error: A fatal internal error occurred, see debug.log for details"), // Same message as AbortNode - "", CClientUIInterface::MSG_ERROR); - StartShutdown(); - return; - } - } else { - strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]; - } - - assert(rpc_io_service == NULL); - rpc_io_service = new boost::asio::io_service(); - rpc_ssl_context = new ssl::context(*rpc_io_service, ssl::context::sslv23); - - const bool fUseSSL = GetBoolArg("-rpcssl", false); - - if (fUseSSL) - { - rpc_ssl_context->set_options(ssl::context::no_sslv2 | ssl::context::no_sslv3); - - boost::filesystem::path pathCertFile(GetArg("-rpcsslcertificatechainfile", "server.cert")); - if (!pathCertFile.is_complete()) pathCertFile = boost::filesystem::path(GetDataDir()) / pathCertFile; - if (boost::filesystem::exists(pathCertFile)) rpc_ssl_context->use_certificate_chain_file(pathCertFile.string()); - else LogPrintf("ThreadRPCServer ERROR: missing server certificate file %s\n", pathCertFile.string()); - - boost::filesystem::path pathPKFile(GetArg("-rpcsslprivatekeyfile", "server.pem")); - if (!pathPKFile.is_complete()) pathPKFile = boost::filesystem::path(GetDataDir()) / pathPKFile; - if (boost::filesystem::exists(pathPKFile)) rpc_ssl_context->use_private_key_file(pathPKFile.string(), ssl::context::pem); - else LogPrintf("ThreadRPCServer ERROR: missing server private key file %s\n", pathPKFile.string()); - - string strCiphers = GetArg("-rpcsslciphers", "TLSv1.2+HIGH:TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!3DES:@STRENGTH"); - SSL_CTX_set_cipher_list(rpc_ssl_context->impl(), strCiphers.c_str()); - } - - std::vector<ip::tcp::endpoint> vEndpoints; - bool bBindAny = false; - int defaultPort = GetArg("-rpcport", BaseParams().RPCPort()); - if (!mapArgs.count("-rpcallowip")) // Default to loopback if not allowing external IPs - { - vEndpoints.push_back(ip::tcp::endpoint(boost::asio::ip::address_v6::loopback(), defaultPort)); - vEndpoints.push_back(ip::tcp::endpoint(boost::asio::ip::address_v4::loopback(), defaultPort)); - if (mapArgs.count("-rpcbind")) - { - LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n"); - } - } else if (mapArgs.count("-rpcbind")) // Specific bind address - { - BOOST_FOREACH(const std::string &addr, mapMultiArgs["-rpcbind"]) - { - try { - vEndpoints.push_back(ParseEndpoint(addr, defaultPort)); - } - catch (const boost::system::system_error&) - { - uiInterface.ThreadSafeMessageBox( - strprintf(_("Could not parse -rpcbind value %s as network address"), addr), - "", CClientUIInterface::MSG_ERROR); - StartShutdown(); - return; - } - } - } else { // No specific bind address specified, bind to any - vEndpoints.push_back(ip::tcp::endpoint(boost::asio::ip::address_v6::any(), defaultPort)); - vEndpoints.push_back(ip::tcp::endpoint(boost::asio::ip::address_v4::any(), defaultPort)); - // Prefer making the socket dual IPv6/IPv4 instead of binding - // to both addresses separately. - bBindAny = true; - } - - bool fListening = false; - std::string strerr; - std::string straddress; - BOOST_FOREACH(const ip::tcp::endpoint &endpoint, vEndpoints) - { - try { - boost::asio::ip::address bindAddress = endpoint.address(); - straddress = bindAddress.to_string(); - LogPrintf("Binding RPC on address %s port %i (IPv4+IPv6 bind any: %i)\n", straddress, endpoint.port(), bBindAny); - boost::system::error_code v6_only_error; - boost::shared_ptr<ip::tcp::acceptor> acceptor(new ip::tcp::acceptor(*rpc_io_service)); - - acceptor->open(endpoint.protocol()); - acceptor->set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); - - // Try making the socket dual IPv6/IPv4 when listening on the IPv6 "any" address - acceptor->set_option(boost::asio::ip::v6_only( - !bBindAny || bindAddress != boost::asio::ip::address_v6::any()), v6_only_error); - - acceptor->bind(endpoint); - acceptor->listen(socket_base::max_connections); - - RPCListen(acceptor, *rpc_ssl_context, fUseSSL); - - fListening = true; - rpc_acceptors.push_back(acceptor); - // If dual IPv6/IPv4 bind successful, skip binding to IPv4 separately - if(bBindAny && bindAddress == boost::asio::ip::address_v6::any() && !v6_only_error) - break; - } - catch (const boost::system::system_error& e) - { - LogPrintf("ERROR: Binding RPC on address %s port %i failed: %s\n", straddress, endpoint.port(), e.what()); - strerr = strprintf(_("An error occurred while setting up the RPC address %s port %u for listening: %s"), straddress, endpoint.port(), e.what()); - } - } - - if (!fListening) { - uiInterface.ThreadSafeMessageBox(strerr, "", CClientUIInterface::MSG_ERROR); - StartShutdown(); - return; - } - - rpc_worker_group = new boost::thread_group(); - for (int i = 0; i < GetArg("-rpcthreads", 4); i++) - rpc_worker_group->create_thread(boost::bind(&boost::asio::io_service::run, rpc_io_service)); + LogPrint("rpc", "Starting RPC\n"); fRPCRunning = true; g_rpcSignals.Started(); + return true; } -void StartDummyRPCThread() +void InterruptRPC() { - if(rpc_io_service == NULL) - { - rpc_io_service = new boost::asio::io_service(); - /* Create dummy "work" to keep the thread from exiting when no timeouts active, - * see http://www.boost.org/doc/libs/1_51_0/doc/html/boost_asio/reference/io_service.html#boost_asio.reference.io_service.stopping_the_io_service_from_running_out_of_work */ - rpc_dummy_work = new boost::asio::io_service::work(*rpc_io_service); - rpc_worker_group = new boost::thread_group(); - rpc_worker_group->create_thread(boost::bind(&boost::asio::io_service::run, rpc_io_service)); - fRPCRunning = true; - } + LogPrint("rpc", "Interrupting RPC\n"); + // Interrupt e.g. running longpolls + fRPCRunning = false; } -void StopRPCThreads() +void StopRPC() { - if (rpc_io_service == NULL) return; - // Set this to false first, so that longpolling loops will exit when woken up - fRPCRunning = false; - - // First, cancel all timers and acceptors - // This is not done automatically by ->stop(), and in some cases the destructor of - // boost::asio::io_service can hang if this is skipped. - boost::system::error_code ec; - BOOST_FOREACH(const boost::shared_ptr<ip::tcp::acceptor> &acceptor, rpc_acceptors) - { - acceptor->cancel(ec); - if (ec) - LogPrintf("%s: Warning: %s when cancelling acceptor\n", __func__, ec.message()); - } - rpc_acceptors.clear(); - BOOST_FOREACH(const PAIRTYPE(std::string, boost::shared_ptr<deadline_timer>) &timer, deadlineTimers) - { - timer.second->cancel(ec); - if (ec) - LogPrintf("%s: Warning: %s when cancelling timer\n", __func__, ec.message()); - } + LogPrint("rpc", "Stopping RPC\n"); deadlineTimers.clear(); - - DeleteAuthCookie(); - - rpc_io_service->stop(); g_rpcSignals.Stopped(); - if (rpc_worker_group != NULL) - rpc_worker_group->join_all(); - delete rpc_dummy_work; rpc_dummy_work = NULL; - delete rpc_worker_group; rpc_worker_group = NULL; - delete rpc_ssl_context; rpc_ssl_context = NULL; - delete rpc_io_service; rpc_io_service = NULL; } bool IsRPCRunning() @@ -802,36 +440,6 @@ bool RPCIsInWarmup(std::string *outStatus) return fRPCInWarmup; } -void RPCRunHandler(const boost::system::error_code& err, boost::function<void(void)> func) -{ - if (!err) - func(); -} - -void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds) -{ - assert(rpc_io_service != NULL); - - if (deadlineTimers.count(name) == 0) - { - deadlineTimers.insert(make_pair(name, - boost::shared_ptr<deadline_timer>(new deadline_timer(*rpc_io_service)))); - } - deadlineTimers[name]->expires_from_now(boost::posix_time::seconds(nSeconds)); - deadlineTimers[name]->async_wait(boost::bind(RPCRunHandler, _1, func)); -} - -class JSONRequest -{ -public: - UniValue id; - string strMethod; - UniValue params; - - JSONRequest() { id = NullUniValue; } - void parse(const UniValue& valRequest); -}; - void JSONRequest::parse(const UniValue& valRequest) { // Parse request @@ -862,7 +470,6 @@ void JSONRequest::parse(const UniValue& valRequest) throw JSONRPCError(RPC_INVALID_REQUEST, "Params must be an array"); } - static UniValue JSONRPCExecOne(const UniValue& req) { UniValue rpc_result(UniValue::VOBJ); @@ -887,7 +494,7 @@ static UniValue JSONRPCExecOne(const UniValue& req) return rpc_result; } -static string JSONRPCExecBatch(const UniValue& vReq) +std::string JSONRPCExecBatch(const UniValue& vReq) { UniValue ret(UniValue::VARR); for (unsigned int reqIdx = 0; reqIdx < vReq.size(); reqIdx++) @@ -896,107 +503,6 @@ static string JSONRPCExecBatch(const UniValue& vReq) return ret.write() + "\n"; } -static bool HTTPReq_JSONRPC(AcceptedConnection *conn, - string& strRequest, - map<string, string>& mapHeaders, - bool fRun) -{ - // Check authorization - if (mapHeaders.count("authorization") == 0) - { - conn->stream() << HTTPError(HTTP_UNAUTHORIZED, false) << std::flush; - return false; - } - - if (!HTTPAuthorized(mapHeaders)) - { - LogPrintf("ThreadRPCServer incorrect password attempt from %s\n", conn->peer_address_to_string()); - /* Deter brute-forcing - We don't support exposing the RPC port, so this shouldn't result - in a DoS. */ - MilliSleep(250); - - conn->stream() << HTTPError(HTTP_UNAUTHORIZED, false) << std::flush; - return false; - } - - JSONRequest jreq; - try - { - // Parse request - UniValue valRequest; - if (!valRequest.read(strRequest)) - throw JSONRPCError(RPC_PARSE_ERROR, "Parse error"); - - string strReply; - - // singleton request - if (valRequest.isObject()) { - jreq.parse(valRequest); - - UniValue result = tableRPC.execute(jreq.strMethod, jreq.params); - - // Send reply - strReply = JSONRPCReply(result, NullUniValue, jreq.id); - - // array of requests - } else if (valRequest.isArray()) - strReply = JSONRPCExecBatch(valRequest.get_array()); - else - throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error"); - - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, strReply.size()) << strReply << std::flush; - } - catch (const UniValue& objError) - { - ErrorReply(conn->stream(), objError, jreq.id); - return false; - } - catch (const std::exception& e) - { - ErrorReply(conn->stream(), JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); - return false; - } - return true; -} - -void ServiceConnection(AcceptedConnection *conn) -{ - bool fRun = true; - while (fRun && !ShutdownRequested()) - { - int nProto = 0; - map<string, string> mapHeaders; - string strRequest, strMethod, strURI; - - // Read HTTP request line - if (!ReadHTTPRequestLine(conn->stream(), nProto, strMethod, strURI)) - break; - - // Read HTTP message headers and body - ReadHTTPMessage(conn->stream(), mapHeaders, strRequest, nProto, MAX_SIZE); - - // HTTP Keep-Alive is false; close connection immediately - if ((mapHeaders["connection"] == "close") || (!GetBoolArg("-rpckeepalive", true))) - fRun = false; - - // Process via JSON-RPC API - if (strURI == "/") { - if (!HTTPReq_JSONRPC(conn, strRequest, mapHeaders, fRun)) - break; - - // Process via HTTP REST API - } else if (strURI.substr(0, 6) == "/rest/" && GetBoolArg("-rest", false)) { - if (!HTTPReq_REST(conn, strURI, strRequest, mapHeaders, fRun)) - break; - - } else { - conn->stream() << HTTPError(HTTP_NOT_FOUND, false) << std::flush; - break; - } - } -} - UniValue CRPCTable::execute(const std::string &strMethod, const UniValue ¶ms) const { // Return immediately if in warmup @@ -1037,4 +543,26 @@ std::string HelpExampleRpc(const std::string& methodname, const std::string& arg "\"method\": \"" + methodname + "\", \"params\": [" + args + "] }' -H 'content-type: text/plain;' http://127.0.0.1:8332/\n"; } +void RPCRegisterTimerInterface(RPCTimerInterface *iface) +{ + timerInterfaces.push_back(iface); +} + +void RPCUnregisterTimerInterface(RPCTimerInterface *iface) +{ + std::vector<RPCTimerInterface*>::iterator i = std::find(timerInterfaces.begin(), timerInterfaces.end(), iface); + assert(i != timerInterfaces.end()); + timerInterfaces.erase(i); +} + +void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds) +{ + if (timerInterfaces.empty()) + throw JSONRPCError(RPC_INTERNAL_ERROR, "No timer handler registered for RPC"); + deadlineTimers.erase(name); + RPCTimerInterface* timerInterface = timerInterfaces[0]; + LogPrint("rpc", "queue run of timer %s in %i seconds (using %s)\n", name, nSeconds, timerInterface->Name()); + deadlineTimers.insert(std::make_pair(name, timerInterface->NewTimer(func, nSeconds*1000))); +} + const CRPCTable tableRPC; diff --git a/src/rpcserver.h b/src/rpcserver.h index 3a71fd510f..83cc37918b 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -32,26 +32,17 @@ namespace RPCServer class CBlockIndex; class CNetAddr; -class AcceptedConnection +class JSONRequest { public: - virtual ~AcceptedConnection() {} + UniValue id; + std::string strMethod; + UniValue params; - virtual std::iostream& stream() = 0; - virtual std::string peer_address_to_string() const = 0; - virtual void close() = 0; + JSONRequest() { id = NullUniValue; } + void parse(const UniValue& valRequest); }; -/** Start RPC threads */ -void StartRPCThreads(); -/** - * Alternative to StartRPCThreads for the GUI, when no server is - * used. The RPC thread in this case is only used to handle timeouts. - * If real RPC threads have already been started this is a no-op. - */ -void StartDummyRPCThread(); -/** Stop RPC threads */ -void StopRPCThreads(); /** Query whether RPC is running */ bool IsRPCRunning(); @@ -81,15 +72,45 @@ void RPCTypeCheck(const UniValue& params, void RPCTypeCheckObj(const UniValue& o, const std::map<std::string, UniValue::VType>& typesExpected, bool fAllowNull=false); +/** Opaque base class for timers returned by NewTimerFunc. + * This provides no methods at the moment, but makes sure that delete + * cleans up the whole state. + */ +class RPCTimerBase +{ +public: + virtual ~RPCTimerBase() {} +}; + /** - * Run func nSeconds from now. Uses boost deadline timers. + * RPC timer "driver". + */ +class RPCTimerInterface +{ +public: + virtual ~RPCTimerInterface() {} + /** Implementation name */ + virtual const char *Name() = 0; + /** Factory function for timers. + * RPC will call the function to create a timer that will call func in *millis* milliseconds. + * @note As the RPC mechanism is backend-neutral, it can use different implementations of timers. + * This is needed to cope with the case in which there is no HTTP server, but + * only GUI RPC console, and to break the dependency of pcserver on httprpc. + */ + virtual RPCTimerBase* NewTimer(boost::function<void(void)>& func, int64_t millis) = 0; +}; + +/** Register factory function for timers */ +void RPCRegisterTimerInterface(RPCTimerInterface *iface); +/** Unregister factory function for timers */ +void RPCUnregisterTimerInterface(RPCTimerInterface *iface); + +/** + * Run func nSeconds from now. * Overrides previous timer <name> (if any). */ void RPCRunLater(const std::string& name, boost::function<void(void)> func, int64_t nSeconds); -//! Convert boost::asio address to CNetAddr -extern CNetAddr BoostAsioToCNetAddr(boost::asio::ip::address address); - typedef UniValue(*rpcfn_type)(const UniValue& params, bool fHelp); class CRPCCommand @@ -134,9 +155,6 @@ extern uint256 ParseHashO(const UniValue& o, std::string strKey); extern std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); extern std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); -extern void InitRPCMining(); -extern void ShutdownRPCMining(); - extern int64_t nWalletUnlockTime; extern CAmount AmountFromValue(const UniValue& value); extern UniValue ValueFromAmount(const CAmount& amount); @@ -244,11 +262,9 @@ extern UniValue getchaintips(const UniValue& params, bool fHelp); extern UniValue invalidateblock(const UniValue& params, bool fHelp); extern UniValue reconsiderblock(const UniValue& params, bool fHelp); -// in rest.cpp -extern bool HTTPReq_REST(AcceptedConnection *conn, - const std::string& strURI, - const std::string& strRequest, - const std::map<std::string, std::string>& mapHeaders, - bool fRun); +bool StartRPC(); +void InterruptRPC(); +void StopRPC(); +std::string JSONRPCExecBatch(const UniValue& vReq); #endif // BITCOIN_RPCSERVER_H diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 06115f5619..184ddc28ab 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -4,9 +4,10 @@ #include "scheduler.h" +#include "reverselock.h" + #include <assert.h> #include <boost/bind.hpp> -#include <boost/thread/reverse_lock.hpp> #include <utility> CScheduler::CScheduler() : nThreadsServicingQueue(0), stopRequested(false), stopWhenEmpty(false) @@ -69,7 +70,7 @@ void CScheduler::serviceQueue() { // Unlock before calling f, so it can reschedule itself or another task // without deadlocking: - boost::reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); + reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); f(); } } catch (...) { diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 2439689d7f..5bf1e98e8f 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -9,6 +9,7 @@ #include <boost/test/unit_test.hpp> #include <list> +#include <vector> BOOST_FIXTURE_TEST_SUITE(mempool_tests, TestingSetup) @@ -100,4 +101,184 @@ BOOST_AUTO_TEST_CASE(MempoolRemoveTest) removed.clear(); } +void CheckSort(CTxMemPool &pool, std::vector<std::string> &sortedOrder) +{ + BOOST_CHECK_EQUAL(pool.size(), sortedOrder.size()); + CTxMemPool::indexed_transaction_set::nth_index<1>::type::iterator it = pool.mapTx.get<1>().begin(); + int count=0; + for (; it != pool.mapTx.get<1>().end(); ++it, ++count) { + BOOST_CHECK_EQUAL(it->GetTx().GetHash().ToString(), sortedOrder[count]); + } +} + +BOOST_AUTO_TEST_CASE(MempoolIndexingTest) +{ + CTxMemPool pool(CFeeRate(0)); + + /* 3rd highest fee */ + CMutableTransaction tx1 = CMutableTransaction(); + tx1.vout.resize(1); + tx1.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx1.vout[0].nValue = 10 * COIN; + pool.addUnchecked(tx1.GetHash(), CTxMemPoolEntry(tx1, 10000LL, 0, 10.0, 1, true)); + + /* highest fee */ + CMutableTransaction tx2 = CMutableTransaction(); + tx2.vout.resize(1); + tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx2.vout[0].nValue = 2 * COIN; + pool.addUnchecked(tx2.GetHash(), CTxMemPoolEntry(tx2, 20000LL, 0, 9.0, 1, true)); + + /* lowest fee */ + CMutableTransaction tx3 = CMutableTransaction(); + tx3.vout.resize(1); + tx3.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx3.vout[0].nValue = 5 * COIN; + pool.addUnchecked(tx3.GetHash(), CTxMemPoolEntry(tx3, 0LL, 0, 100.0, 1, true)); + + /* 2nd highest fee */ + CMutableTransaction tx4 = CMutableTransaction(); + tx4.vout.resize(1); + tx4.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx4.vout[0].nValue = 6 * COIN; + pool.addUnchecked(tx4.GetHash(), CTxMemPoolEntry(tx4, 15000LL, 0, 1.0, 1, true)); + + /* equal fee rate to tx1, but newer */ + CMutableTransaction tx5 = CMutableTransaction(); + tx5.vout.resize(1); + tx5.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx5.vout[0].nValue = 11 * COIN; + pool.addUnchecked(tx5.GetHash(), CTxMemPoolEntry(tx5, 10000LL, 1, 10.0, 1, true)); + BOOST_CHECK_EQUAL(pool.size(), 5); + + std::vector<std::string> sortedOrder; + sortedOrder.resize(5); + sortedOrder[0] = tx2.GetHash().ToString(); // 20000 + sortedOrder[1] = tx4.GetHash().ToString(); // 15000 + sortedOrder[2] = tx1.GetHash().ToString(); // 10000 + sortedOrder[3] = tx5.GetHash().ToString(); // 10000 + sortedOrder[4] = tx3.GetHash().ToString(); // 0 + CheckSort(pool, sortedOrder); + + /* low fee but with high fee child */ + /* tx6 -> tx7 -> tx8, tx9 -> tx10 */ + CMutableTransaction tx6 = CMutableTransaction(); + tx6.vout.resize(1); + tx6.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx6.vout[0].nValue = 20 * COIN; + pool.addUnchecked(tx6.GetHash(), CTxMemPoolEntry(tx6, 0LL, 1, 10.0, 1, true)); + BOOST_CHECK_EQUAL(pool.size(), 6); + // Check that at this point, tx6 is sorted low + sortedOrder.push_back(tx6.GetHash().ToString()); + CheckSort(pool, sortedOrder); + + CTxMemPool::setEntries setAncestors; + setAncestors.insert(pool.mapTx.find(tx6.GetHash())); + CMutableTransaction tx7 = CMutableTransaction(); + tx7.vin.resize(1); + tx7.vin[0].prevout = COutPoint(tx6.GetHash(), 0); + tx7.vin[0].scriptSig = CScript() << OP_11; + tx7.vout.resize(2); + tx7.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx7.vout[0].nValue = 10 * COIN; + tx7.vout[1].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx7.vout[1].nValue = 1 * COIN; + + CTxMemPool::setEntries setAncestorsCalculated; + std::string dummy; + CTxMemPoolEntry entry7(tx7, 2000000LL, 1, 10.0, 1, true); + BOOST_CHECK_EQUAL(pool.CalculateMemPoolAncestors(entry7, setAncestorsCalculated, 100, 1000000, 1000, 1000000, dummy), true); + BOOST_CHECK(setAncestorsCalculated == setAncestors); + + pool.addUnchecked(tx7.GetHash(), CTxMemPoolEntry(tx7, 2000000LL, 1, 10.0, 1, true), setAncestors); + BOOST_CHECK_EQUAL(pool.size(), 7); + + // Now tx6 should be sorted higher (high fee child): tx7, tx6, tx2, ... + sortedOrder.erase(sortedOrder.end()-1); + sortedOrder.insert(sortedOrder.begin(), tx6.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin(), tx7.GetHash().ToString()); + CheckSort(pool, sortedOrder); + + /* low fee child of tx7 */ + CMutableTransaction tx8 = CMutableTransaction(); + tx8.vin.resize(1); + tx8.vin[0].prevout = COutPoint(tx7.GetHash(), 0); + tx8.vin[0].scriptSig = CScript() << OP_11; + tx8.vout.resize(1); + tx8.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx8.vout[0].nValue = 10 * COIN; + setAncestors.insert(pool.mapTx.find(tx7.GetHash())); + pool.addUnchecked(tx8.GetHash(), CTxMemPoolEntry(tx8, 0LL, 2, 10.0, 1, true), setAncestors); + + // Now tx8 should be sorted low, but tx6/tx both high + sortedOrder.push_back(tx8.GetHash().ToString()); + CheckSort(pool, sortedOrder); + + /* low fee child of tx7 */ + CMutableTransaction tx9 = CMutableTransaction(); + tx9.vin.resize(1); + tx9.vin[0].prevout = COutPoint(tx7.GetHash(), 1); + tx9.vin[0].scriptSig = CScript() << OP_11; + tx9.vout.resize(1); + tx9.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx9.vout[0].nValue = 1 * COIN; + pool.addUnchecked(tx9.GetHash(), CTxMemPoolEntry(tx9, 0LL, 3, 10.0, 1, true), setAncestors); + + // tx9 should be sorted low + BOOST_CHECK_EQUAL(pool.size(), 9); + sortedOrder.push_back(tx9.GetHash().ToString()); + CheckSort(pool, sortedOrder); + + std::vector<std::string> snapshotOrder = sortedOrder; + + setAncestors.insert(pool.mapTx.find(tx8.GetHash())); + setAncestors.insert(pool.mapTx.find(tx9.GetHash())); + /* tx10 depends on tx8 and tx9 and has a high fee*/ + CMutableTransaction tx10 = CMutableTransaction(); + tx10.vin.resize(2); + tx10.vin[0].prevout = COutPoint(tx8.GetHash(), 0); + tx10.vin[0].scriptSig = CScript() << OP_11; + tx10.vin[1].prevout = COutPoint(tx9.GetHash(), 0); + tx10.vin[1].scriptSig = CScript() << OP_11; + tx10.vout.resize(1); + tx10.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx10.vout[0].nValue = 10 * COIN; + + setAncestorsCalculated.clear(); + CTxMemPoolEntry entry10(tx10, 200000LL, 4, 10.0, 1, true); + BOOST_CHECK_EQUAL(pool.CalculateMemPoolAncestors(entry10, setAncestorsCalculated, 100, 1000000, 1000, 1000000, dummy), true); + BOOST_CHECK(setAncestorsCalculated == setAncestors); + + pool.addUnchecked(tx10.GetHash(), CTxMemPoolEntry(tx10, 200000LL, 4, 10.0, 1, true), setAncestors); + + /** + * tx8 and tx9 should both now be sorted higher + * Final order after tx10 is added: + * + * tx7 = 2.2M (4 txs) + * tx6 = 2.2M (5 txs) + * tx10 = 200k (1 tx) + * tx8 = 200k (2 txs) + * tx9 = 200k (2 txs) + * tx2 = 20000 (1) + * tx4 = 15000 (1) + * tx1 = 10000 (1) + * tx5 = 10000 (1) + * tx3 = 0 (1) + */ + sortedOrder.erase(sortedOrder.end()-2, sortedOrder.end()); // take out tx8, tx9 from the end + sortedOrder.insert(sortedOrder.begin()+2, tx10.GetHash().ToString()); // tx10 is after tx6 + sortedOrder.insert(sortedOrder.begin()+3, tx9.GetHash().ToString()); + sortedOrder.insert(sortedOrder.begin()+3, tx8.GetHash().ToString()); + CheckSort(pool, sortedOrder); + + // there should be 10 transactions in the mempool + BOOST_CHECK_EQUAL(pool.size(), 10); + + // Now try removing tx10 and verify the sort order returns to normal + std::list<CTransaction> removed; + pool.remove(pool.mapTx.find(tx10.GetHash())->GetTx(), removed, true); + CheckSort(pool, snapshotOrder); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index ad79a558c2..91a3a5738e 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -87,7 +87,7 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) pblock->vtx[0] = CTransaction(txCoinbase); if (txFirst.size() < 2) txFirst.push_back(new CTransaction(pblock->vtx[0])); - pblock->hashMerkleRoot = pblock->BuildMerkleTree(); + pblock->hashMerkleRoot = pblock->ComputeMerkleRoot(); pblock->nNonce = blockinfo[i].nonce; CValidationState state; BOOST_CHECK(ProcessNewBlock(state, NULL, pblock, true, NULL)); diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 3f99dc98d9..b1ef0ed24a 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -149,12 +149,90 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(CSubNet(CNetAddr("127.0.0.1")).IsValid()); BOOST_CHECK(CSubNet(CNetAddr("127.0.0.1")).Match(CNetAddr("127.0.0.1"))); BOOST_CHECK(!CSubNet(CNetAddr("127.0.0.1")).Match(CNetAddr("127.0.0.2"))); - BOOST_CHECK(CSubNet(CNetAddr("127.0.0.1")).ToString() == "127.0.0.1/255.255.255.255"); + BOOST_CHECK(CSubNet(CNetAddr("127.0.0.1")).ToString() == "127.0.0.1/32"); BOOST_CHECK(CSubNet(CNetAddr("1:2:3:4:5:6:7:8")).IsValid()); BOOST_CHECK(CSubNet(CNetAddr("1:2:3:4:5:6:7:8")).Match(CNetAddr("1:2:3:4:5:6:7:8"))); BOOST_CHECK(!CSubNet(CNetAddr("1:2:3:4:5:6:7:8")).Match(CNetAddr("1:2:3:4:5:6:7:9"))); - BOOST_CHECK(CSubNet(CNetAddr("1:2:3:4:5:6:7:8")).ToString() == "1:2:3:4:5:6:7:8/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"); + BOOST_CHECK(CSubNet(CNetAddr("1:2:3:4:5:6:7:8")).ToString() == "1:2:3:4:5:6:7:8/128"); + + CSubNet subnet = CSubNet("1.2.3.4/255.255.255.255"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/32"); + subnet = CSubNet("1.2.3.4/255.255.255.254"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/31"); + subnet = CSubNet("1.2.3.4/255.255.255.252"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/30"); + subnet = CSubNet("1.2.3.4/255.255.255.248"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/29"); + subnet = CSubNet("1.2.3.4/255.255.255.240"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/28"); + subnet = CSubNet("1.2.3.4/255.255.255.224"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/27"); + subnet = CSubNet("1.2.3.4/255.255.255.192"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/26"); + subnet = CSubNet("1.2.3.4/255.255.255.128"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/25"); + subnet = CSubNet("1.2.3.4/255.255.255.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/24"); + subnet = CSubNet("1.2.3.4/255.255.254.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.2.0/23"); + subnet = CSubNet("1.2.3.4/255.255.252.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/22"); + subnet = CSubNet("1.2.3.4/255.255.248.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/21"); + subnet = CSubNet("1.2.3.4/255.255.240.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/20"); + subnet = CSubNet("1.2.3.4/255.255.224.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/19"); + subnet = CSubNet("1.2.3.4/255.255.192.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/18"); + subnet = CSubNet("1.2.3.4/255.255.128.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/17"); + subnet = CSubNet("1.2.3.4/255.255.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/16"); + subnet = CSubNet("1.2.3.4/255.254.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/15"); + subnet = CSubNet("1.2.3.4/255.252.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/14"); + subnet = CSubNet("1.2.3.4/255.248.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/13"); + subnet = CSubNet("1.2.3.4/255.240.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/12"); + subnet = CSubNet("1.2.3.4/255.224.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/11"); + subnet = CSubNet("1.2.3.4/255.192.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/10"); + subnet = CSubNet("1.2.3.4/255.128.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/9"); + subnet = CSubNet("1.2.3.4/255.0.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/8"); + subnet = CSubNet("1.2.3.4/254.0.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/7"); + subnet = CSubNet("1.2.3.4/252.0.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/6"); + subnet = CSubNet("1.2.3.4/248.0.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/5"); + subnet = CSubNet("1.2.3.4/240.0.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/4"); + subnet = CSubNet("1.2.3.4/224.0.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/3"); + subnet = CSubNet("1.2.3.4/192.0.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/2"); + subnet = CSubNet("1.2.3.4/128.0.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/1"); + subnet = CSubNet("1.2.3.4/0.0.0.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/0"); + + subnet = CSubNet("1:2:3:4:5:6:7:8/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1:2:3:4:5:6:7:8/128"); + subnet = CSubNet("1:2:3:4:5:6:7:8/ffff:0000:0000:0000:0000:0000:0000:0000"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1::/16"); + subnet = CSubNet("1:2:3:4:5:6:7:8/0000:0000:0000:0000:0000:0000:0000:0000"); + BOOST_CHECK_EQUAL(subnet.ToString(), "::/0"); + subnet = CSubNet("1.2.3.4/255.255.232.0"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/255.255.232.0"); + subnet = CSubNet("1:2:3:4:5:6:7:8/ffff:ffff:ffff:fffe:ffff:ffff:ffff:ff0f"); + BOOST_CHECK_EQUAL(subnet.ToString(), "1:2:3:4:5:6:7:8/ffff:ffff:ffff:fffe:ffff:ffff:ffff:ff0f"); } BOOST_AUTO_TEST_CASE(netbase_getgroup) diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index f6d06d6805..d9f3c3e467 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -48,7 +48,7 @@ BOOST_AUTO_TEST_CASE(pmt_test1) } // calculate actual merkle root and height - uint256 merkleRoot1 = block.BuildMerkleTree(); + uint256 merkleRoot1 = block.ComputeMerkleRoot(); std::vector<uint256> vTxid(nTx, uint256()); for (unsigned int j=0; j<nTx; j++) vTxid[j] = block.vtx[j].GetHash(); diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp new file mode 100644 index 0000000000..e7e627ae0f --- /dev/null +++ b/src/test/reverselock_tests.cpp @@ -0,0 +1,64 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "reverselock.h" +#include "test/test_bitcoin.h" + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(reverselock_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(reverselock_basics) +{ + boost::mutex mutex; + boost::unique_lock<boost::mutex> lock(mutex); + + BOOST_CHECK(lock.owns_lock()); + { + reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); + BOOST_CHECK(!lock.owns_lock()); + } + BOOST_CHECK(lock.owns_lock()); +} + +BOOST_AUTO_TEST_CASE(reverselock_errors) +{ + boost::mutex mutex; + boost::unique_lock<boost::mutex> lock(mutex); + + // Make sure trying to reverse lock an unlocked lock fails + lock.unlock(); + + BOOST_CHECK(!lock.owns_lock()); + + bool failed = false; + try { + reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); + } catch(...) { + failed = true; + } + + BOOST_CHECK(failed); + BOOST_CHECK(!lock.owns_lock()); + + // Make sure trying to lock a lock after it has been reverse locked fails + failed = false; + bool locked = false; + + lock.lock(); + BOOST_CHECK(lock.owns_lock()); + + try { + reverse_lock<boost::unique_lock<boost::mutex> > rlock(lock); + lock.lock(); + locked = true; + } catch(...) { + failed = true; + } + + BOOST_CHECK(locked && failed); + BOOST_CHECK(lock.owns_lock()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index a65572e6f6..1bd59497ff 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -224,21 +224,6 @@ BOOST_AUTO_TEST_CASE(json_parse_errors) BOOST_CHECK_THROW(ParseNonRFCJSONValue("3J98t1WpEZ73CNmQviecrnyiWrnqRhWNL"), std::runtime_error); } -BOOST_AUTO_TEST_CASE(rpc_boostasiotocnetaddr) -{ - // Check IPv4 addresses - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("1.2.3.4")).ToString(), "1.2.3.4"); - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("127.0.0.1")).ToString(), "127.0.0.1"); - // Check IPv6 addresses - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("::1")).ToString(), "::1"); - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("123:4567:89ab:cdef:123:4567:89ab:cdef")).ToString(), - "123:4567:89ab:cdef:123:4567:89ab:cdef"); - // v4 compatible must be interpreted as IPv4 - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("::0:127.0.0.1")).ToString(), "127.0.0.1"); - // v4 mapped must be interpreted as IPv4 - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("::ffff:127.0.0.1")).ToString(), "127.0.0.1"); -} - BOOST_AUTO_TEST_CASE(rpc_ban) { BOOST_CHECK_NO_THROW(CallRPC(string("clearbanned"))); @@ -250,7 +235,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) UniValue ar = r.get_array(); UniValue o1 = ar[0].get_obj(); UniValue adr = find_value(o1, "address"); - BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/255.255.255.255"); + BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/32"); BOOST_CHECK_NO_THROW(CallRPC(string("setban 127.0.0.0 remove")));; BOOST_CHECK_NO_THROW(r = CallRPC(string("listbanned"))); ar = r.get_array(); @@ -262,7 +247,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) o1 = ar[0].get_obj(); adr = find_value(o1, "address"); UniValue banned_until = find_value(o1, "banned_until"); - BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/255.255.255.0"); + BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/24"); BOOST_CHECK_EQUAL(banned_until.get_int64(), 1607731200); // absolute time check BOOST_CHECK_NO_THROW(CallRPC(string("clearbanned"))); @@ -273,7 +258,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) o1 = ar[0].get_obj(); adr = find_value(o1, "address"); banned_until = find_value(o1, "banned_until"); - BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/255.255.255.0"); + BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/24"); int64_t now = GetTime(); BOOST_CHECK(banned_until.get_int64() > now); BOOST_CHECK(banned_until.get_int64()-now <= 200); @@ -303,15 +288,15 @@ BOOST_AUTO_TEST_CASE(rpc_ban) ar = r.get_array(); o1 = ar[0].get_obj(); adr = find_value(o1, "address"); - BOOST_CHECK_EQUAL(adr.get_str(), "fe80::202:b3ff:fe1e:8329/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"); + BOOST_CHECK_EQUAL(adr.get_str(), "fe80::202:b3ff:fe1e:8329/128"); BOOST_CHECK_NO_THROW(CallRPC(string("clearbanned"))); - BOOST_CHECK_NO_THROW(r = CallRPC(string("setban 2001:db8::/30 add"))); + BOOST_CHECK_NO_THROW(r = CallRPC(string("setban 2001:db8::/ffff:fffc:0:0:0:0:0:0 add"))); BOOST_CHECK_NO_THROW(r = CallRPC(string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); adr = find_value(o1, "address"); - BOOST_CHECK_EQUAL(adr.get_str(), "2001:db8::/ffff:fffc:0:0:0:0:0:0"); + BOOST_CHECK_EQUAL(adr.get_str(), "2001:db8::/30"); BOOST_CHECK_NO_THROW(CallRPC(string("clearbanned"))); BOOST_CHECK_NO_THROW(r = CallRPC(string("setban 2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/128 add"))); @@ -319,7 +304,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) ar = r.get_array(); o1 = ar[0].get_obj(); adr = find_value(o1, "address"); - BOOST_CHECK_EQUAL(adr.get_str(), "2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"); + BOOST_CHECK_EQUAL(adr.get_str(), "2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/128"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index e956cc5b90..bde16a517f 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -413,10 +413,10 @@ BOOST_AUTO_TEST_CASE(test_FormatSubVersion) comments.push_back(std::string("comment1")); std::vector<std::string> comments2; comments2.push_back(std::string("comment1")); - comments2.push_back(std::string("comment2")); + comments2.push_back(SanitizeString(std::string("Comment2; .,_?@; !\"#$%&'()*+-/<=>[]\\^`{|}~"), SAFE_CHARS_UA_COMMENT)); // Semicolon is discouraged but not forbidden by BIP-0014 BOOST_CHECK_EQUAL(FormatSubVersion("Test", 99900, std::vector<std::string>()),std::string("/Test:0.9.99/")); BOOST_CHECK_EQUAL(FormatSubVersion("Test", 99900, comments),std::string("/Test:0.9.99(comment1)/")); - BOOST_CHECK_EQUAL(FormatSubVersion("Test", 99900, comments2),std::string("/Test:0.9.99(comment1; comment2)/")); + BOOST_CHECK_EQUAL(FormatSubVersion("Test", 99900, comments2),std::string("/Test:0.9.99(comment1; Comment2; .,_?@; )/")); } BOOST_AUTO_TEST_CASE(test_ParseFixedPoint) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c921dae45d..1370cab0c0 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -17,12 +17,6 @@ using namespace std; -CTxMemPoolEntry::CTxMemPoolEntry(): - nFee(0), nTxSize(0), nModSize(0), nUsageSize(0), nTime(0), dPriority(0.0), hadNoDependencies(false) -{ - nHeight = MEMPOOL_HEIGHT; -} - CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, int64_t _nTime, double _dPriority, unsigned int _nHeight, bool poolHasNoInputsOf): @@ -32,6 +26,10 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION); nModSize = tx.CalculateModifiedSize(nTxSize); nUsageSize = RecursiveDynamicUsage(tx); + + nCountWithDescendants = 1; + nSizeWithDescendants = nTxSize; + nFeesWithDescendants = nFee; } CTxMemPoolEntry::CTxMemPoolEntry(const CTxMemPoolEntry& other) @@ -48,6 +46,265 @@ CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const return dResult; } +// Update the given tx for any in-mempool descendants. +// Assumes that setMemPoolChildren is correct for the given tx and all +// descendants. +bool CTxMemPool::UpdateForDescendants(txiter updateIt, int maxDescendantsToVisit, cacheMap &cachedDescendants, const std::set<uint256> &setExclude) +{ + // Track the number of entries (outside setExclude) that we'd need to visit + // (will bail out if it exceeds maxDescendantsToVisit) + int nChildrenToVisit = 0; + + setEntries stageEntries, setAllDescendants; + stageEntries = GetMemPoolChildren(updateIt); + + while (!stageEntries.empty()) { + const txiter cit = *stageEntries.begin(); + if (cit->IsDirty()) { + // Don't consider any more children if any descendant is dirty + return false; + } + setAllDescendants.insert(cit); + stageEntries.erase(cit); + const setEntries &setChildren = GetMemPoolChildren(cit); + BOOST_FOREACH(const txiter childEntry, setChildren) { + cacheMap::iterator cacheIt = cachedDescendants.find(childEntry); + if (cacheIt != cachedDescendants.end()) { + // We've already calculated this one, just add the entries for this set + // but don't traverse again. + BOOST_FOREACH(const txiter cacheEntry, cacheIt->second) { + // update visit count only for new child transactions + // (outside of setExclude and stageEntries) + if (setAllDescendants.insert(cacheEntry).second && + !setExclude.count(cacheEntry->GetTx().GetHash()) && + !stageEntries.count(cacheEntry)) { + nChildrenToVisit++; + } + } + } else if (!setAllDescendants.count(childEntry)) { + // Schedule for later processing and update our visit count + if (stageEntries.insert(childEntry).second && !setExclude.count(childEntry->GetTx().GetHash())) { + nChildrenToVisit++; + } + } + if (nChildrenToVisit > maxDescendantsToVisit) { + return false; + } + } + } + // setAllDescendants now contains all in-mempool descendants of updateIt. + // Update and add to cached descendant map + int64_t modifySize = 0; + CAmount modifyFee = 0; + int64_t modifyCount = 0; + BOOST_FOREACH(txiter cit, setAllDescendants) { + if (!setExclude.count(cit->GetTx().GetHash())) { + modifySize += cit->GetTxSize(); + modifyFee += cit->GetFee(); + modifyCount++; + cachedDescendants[updateIt].insert(cit); + } + } + mapTx.modify(updateIt, update_descendant_state(modifySize, modifyFee, modifyCount)); + return true; +} + +// vHashesToUpdate is the set of transaction hashes from a disconnected block +// which has been re-added to the mempool. +// for each entry, look for descendants that are outside hashesToUpdate, and +// add fee/size information for such descendants to the parent. +void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashesToUpdate) +{ + LOCK(cs); + // For each entry in vHashesToUpdate, store the set of in-mempool, but not + // in-vHashesToUpdate transactions, so that we don't have to recalculate + // descendants when we come across a previously seen entry. + cacheMap mapMemPoolDescendantsToUpdate; + + // Use a set for lookups into vHashesToUpdate (these entries are already + // accounted for in the state of their ancestors) + std::set<uint256> setAlreadyIncluded(vHashesToUpdate.begin(), vHashesToUpdate.end()); + + // Iterate in reverse, so that whenever we are looking at at a transaction + // we are sure that all in-mempool descendants have already been processed. + // This maximizes the benefit of the descendant cache and guarantees that + // setMemPoolChildren will be updated, an assumption made in + // UpdateForDescendants. + BOOST_REVERSE_FOREACH(const uint256 &hash, vHashesToUpdate) { + // we cache the in-mempool children to avoid duplicate updates + setEntries setChildren; + // calculate children from mapNextTx + txiter it = mapTx.find(hash); + if (it == mapTx.end()) { + continue; + } + std::map<COutPoint, CInPoint>::iterator iter = mapNextTx.lower_bound(COutPoint(hash, 0)); + // First calculate the children, and update setMemPoolChildren to + // include them, and update their setMemPoolParents to include this tx. + for (; iter != mapNextTx.end() && iter->first.hash == hash; ++iter) { + const uint256 &childHash = iter->second.ptx->GetHash(); + txiter childIter = mapTx.find(childHash); + assert(childIter != mapTx.end()); + // We can skip updating entries we've encountered before or that + // are in the block (which are already accounted for). + if (setChildren.insert(childIter).second && !setAlreadyIncluded.count(childHash)) { + UpdateChild(it, childIter, true); + UpdateParent(childIter, it, true); + } + } + if (!UpdateForDescendants(it, 100, mapMemPoolDescendantsToUpdate, setAlreadyIncluded)) { + // Mark as dirty if we can't do the calculation. + mapTx.modify(it, set_dirty()); + } + } +} + +bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents /* = true */) +{ + setEntries parentHashes; + const CTransaction &tx = entry.GetTx(); + + if (fSearchForParents) { + // Get parents of this transaction that are in the mempool + // GetMemPoolParents() is only valid for entries in the mempool, so we + // iterate mapTx to find parents. + for (unsigned int i = 0; i < tx.vin.size(); i++) { + txiter piter = mapTx.find(tx.vin[i].prevout.hash); + if (piter != mapTx.end()) { + parentHashes.insert(piter); + if (parentHashes.size() + 1 > limitAncestorCount) { + errString = strprintf("too many unconfirmed parents [limit: %u]", limitAncestorCount); + return false; + } + } + } + } else { + // If we're not searching for parents, we require this to be an + // entry in the mempool already. + txiter it = mapTx.iterator_to(entry); + parentHashes = GetMemPoolParents(it); + } + + size_t totalSizeWithAncestors = entry.GetTxSize(); + + while (!parentHashes.empty()) { + txiter stageit = *parentHashes.begin(); + + setAncestors.insert(stageit); + parentHashes.erase(stageit); + totalSizeWithAncestors += stageit->GetTxSize(); + + if (stageit->GetSizeWithDescendants() + entry.GetTxSize() > limitDescendantSize) { + errString = strprintf("exceeds descendant size limit for tx %s [limit: %u]", stageit->GetTx().GetHash().ToString(), limitDescendantSize); + return false; + } else if (stageit->GetCountWithDescendants() + 1 > limitDescendantCount) { + errString = strprintf("too many descendants for tx %s [limit: %u]", stageit->GetTx().GetHash().ToString(), limitDescendantCount); + return false; + } else if (totalSizeWithAncestors > limitAncestorSize) { + errString = strprintf("exceeds ancestor size limit [limit: %u]", limitAncestorSize); + return false; + } + + const setEntries & setMemPoolParents = GetMemPoolParents(stageit); + BOOST_FOREACH(const txiter &phash, setMemPoolParents) { + // If this is a new ancestor, add it. + if (setAncestors.count(phash) == 0) { + parentHashes.insert(phash); + } + if (parentHashes.size() + setAncestors.size() + 1 > limitAncestorCount) { + errString = strprintf("too many unconfirmed ancestors [limit: %u]", limitAncestorCount); + return false; + } + } + } + + return true; +} + +void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors) +{ + setEntries parentIters = GetMemPoolParents(it); + // add or remove this tx as a child of each parent + BOOST_FOREACH(txiter piter, parentIters) { + UpdateChild(piter, it, add); + } + const int64_t updateCount = (add ? 1 : -1); + const int64_t updateSize = updateCount * it->GetTxSize(); + const CAmount updateFee = updateCount * it->GetFee(); + BOOST_FOREACH(txiter ancestorIt, setAncestors) { + mapTx.modify(ancestorIt, update_descendant_state(updateSize, updateFee, updateCount)); + } +} + +void CTxMemPool::UpdateChildrenForRemoval(txiter it) +{ + const setEntries &setMemPoolChildren = GetMemPoolChildren(it); + BOOST_FOREACH(txiter updateIt, setMemPoolChildren) { + UpdateParent(updateIt, it, false); + } +} + +void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove) +{ + // For each entry, walk back all ancestors and decrement size associated with this + // transaction + const uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); + BOOST_FOREACH(txiter removeIt, entriesToRemove) { + setEntries setAncestors; + const CTxMemPoolEntry &entry = *removeIt; + std::string dummy; + // Since this is a tx that is already in the mempool, we can call CMPA + // with fSearchForParents = false. If the mempool is in a consistent + // state, then using true or false should both be correct, though false + // should be a bit faster. + // However, if we happen to be in the middle of processing a reorg, then + // the mempool can be in an inconsistent state. In this case, the set + // of ancestors reachable via mapLinks will be the same as the set of + // ancestors whose packages include this transaction, because when we + // add a new transaction to the mempool in addUnchecked(), we assume it + // has no children, and in the case of a reorg where that assumption is + // false, the in-mempool children aren't linked to the in-block tx's + // until UpdateTransactionsFromBlock() is called. + // So if we're being called during a reorg, ie before + // UpdateTransactionsFromBlock() has been called, then mapLinks[] will + // differ from the set of mempool parents we'd calculate by searching, + // and it's important that we use the mapLinks[] notion of ancestor + // transactions as the set of things to update for removal. + CalculateMemPoolAncestors(entry, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy, false); + // Note that UpdateAncestorsOf severs the child links that point to + // removeIt in the entries for the parents of removeIt. This is + // fine since we don't need to use the mempool children of any entries + // to walk back over our ancestors (but we do need the mempool + // parents!) + UpdateAncestorsOf(false, removeIt, setAncestors); + } + // After updating all the ancestor sizes, we can now sever the link between each + // transaction being removed and any mempool children (ie, update setMemPoolParents + // for each direct child of a transaction being removed). + BOOST_FOREACH(txiter removeIt, entriesToRemove) { + UpdateChildrenForRemoval(removeIt); + } +} + +void CTxMemPoolEntry::SetDirty() +{ + nCountWithDescendants = 0; + nSizeWithDescendants = nTxSize; + nFeesWithDescendants = nFee; +} + +void CTxMemPoolEntry::UpdateState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount) +{ + if (!IsDirty()) { + nSizeWithDescendants += modifySize; + assert(int64_t(nSizeWithDescendants) > 0); + nFeesWithDescendants += modifyFee; + assert(nFeesWithDescendants >= 0); + nCountWithDescendants += modifyCount; + assert(int64_t(nCountWithDescendants) > 0); + } +} + CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee) : nTransactionsUpdated(0) { @@ -89,34 +346,103 @@ void CTxMemPool::AddTransactionsUpdated(unsigned int n) nTransactionsUpdated += n; } - -bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate) +bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool fCurrentEstimate) { // Add to memory pool without checking anything. // Used by main.cpp AcceptToMemoryPool(), which DOES do // all the appropriate checks. LOCK(cs); - mapTx[hash] = entry; - const CTransaction& tx = mapTx[hash].GetTx(); - for (unsigned int i = 0; i < tx.vin.size(); i++) + indexed_transaction_set::iterator newit = mapTx.insert(entry).first; + mapLinks.insert(make_pair(newit, TxLinks())); + + // Update cachedInnerUsage to include contained transaction's usage. + // (When we update the entry for in-mempool parents, memory usage will be + // further updated.) + cachedInnerUsage += entry.DynamicMemoryUsage(); + + const CTransaction& tx = newit->GetTx(); + std::set<uint256> setParentTransactions; + for (unsigned int i = 0; i < tx.vin.size(); i++) { mapNextTx[tx.vin[i].prevout] = CInPoint(&tx, i); + setParentTransactions.insert(tx.vin[i].prevout.hash); + } + // Don't bother worrying about child transactions of this one. + // Normal case of a new transaction arriving is that there can't be any + // children, because such children would be orphans. + // An exception to that is if a transaction enters that used to be in a block. + // In that case, our disconnect block logic will call UpdateTransactionsFromBlock + // to clean up the mess we're leaving here. + + // Update ancestors with information about this tx + BOOST_FOREACH (const uint256 &phash, setParentTransactions) { + txiter pit = mapTx.find(phash); + if (pit != mapTx.end()) { + UpdateParent(newit, pit, true); + } + } + UpdateAncestorsOf(true, newit, setAncestors); + nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); - cachedInnerUsage += entry.DynamicMemoryUsage(); minerPolicyEstimator->processTransaction(entry, fCurrentEstimate); return true; } +void CTxMemPool::removeUnchecked(txiter it) +{ + const uint256 hash = it->GetTx().GetHash(); + BOOST_FOREACH(const CTxIn& txin, it->GetTx().vin) + mapNextTx.erase(txin.prevout); + + totalTxSize -= it->GetTxSize(); + cachedInnerUsage -= it->DynamicMemoryUsage(); + cachedInnerUsage -= memusage::DynamicUsage(mapLinks[it].parents) + memusage::DynamicUsage(mapLinks[it].children); + mapLinks.erase(it); + mapTx.erase(it); + nTransactionsUpdated++; + minerPolicyEstimator->removeTx(hash); +} + +// Calculates descendants of entry that are not already in setDescendants, and adds to +// setDescendants. Assumes entryit is already a tx in the mempool and setMemPoolChildren +// is correct for tx and all descendants. +// Also assumes that if an entry is in setDescendants already, then all +// in-mempool descendants of it are already in setDescendants as well, so that we +// can save time by not iterating over those entries. +void CTxMemPool::CalculateDescendants(txiter entryit, setEntries &setDescendants) +{ + setEntries stage; + if (setDescendants.count(entryit) == 0) { + stage.insert(entryit); + } + // Traverse down the children of entry, only adding children that are not + // accounted for in setDescendants already (because those children have either + // already been walked, or will be walked in this iteration). + while (!stage.empty()) { + txiter it = *stage.begin(); + setDescendants.insert(it); + stage.erase(it); + + const setEntries &setChildren = GetMemPoolChildren(it); + BOOST_FOREACH(const txiter &childiter, setChildren) { + if (!setDescendants.count(childiter)) { + stage.insert(childiter); + } + } + } +} void CTxMemPool::remove(const CTransaction &origTx, std::list<CTransaction>& removed, bool fRecursive) { // Remove transaction from memory pool { LOCK(cs); - std::deque<uint256> txToRemove; - txToRemove.push_back(origTx.GetHash()); - if (fRecursive && !mapTx.count(origTx.GetHash())) { + setEntries txToRemove; + txiter origit = mapTx.find(origTx.GetHash()); + if (origit != mapTx.end()) { + txToRemove.insert(origit); + } else if (fRecursive) { // If recursively removing but origTx isn't in the mempool // be sure to remove any children that are in the pool. This can // happen during chain re-orgs if origTx isn't re-accepted into @@ -125,34 +451,23 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list<CTransaction>& rem std::map<COutPoint, CInPoint>::iterator it = mapNextTx.find(COutPoint(origTx.GetHash(), i)); if (it == mapNextTx.end()) continue; - txToRemove.push_back(it->second.ptx->GetHash()); + txiter nextit = mapTx.find(it->second.ptx->GetHash()); + assert(nextit != mapTx.end()); + txToRemove.insert(nextit); } } - while (!txToRemove.empty()) - { - uint256 hash = txToRemove.front(); - txToRemove.pop_front(); - if (!mapTx.count(hash)) - continue; - const CTransaction& tx = mapTx[hash].GetTx(); - if (fRecursive) { - for (unsigned int i = 0; i < tx.vout.size(); i++) { - std::map<COutPoint, CInPoint>::iterator it = mapNextTx.find(COutPoint(hash, i)); - if (it == mapNextTx.end()) - continue; - txToRemove.push_back(it->second.ptx->GetHash()); - } + setEntries setAllRemoves; + if (fRecursive) { + BOOST_FOREACH(txiter it, txToRemove) { + CalculateDescendants(it, setAllRemoves); } - BOOST_FOREACH(const CTxIn& txin, tx.vin) - mapNextTx.erase(txin.prevout); - - removed.push_back(tx); - totalTxSize -= mapTx[hash].GetTxSize(); - cachedInnerUsage -= mapTx[hash].DynamicMemoryUsage(); - mapTx.erase(hash); - nTransactionsUpdated++; - minerPolicyEstimator->removeTx(hash); + } else { + setAllRemoves.swap(txToRemove); } + BOOST_FOREACH(txiter it, setAllRemoves) { + removed.push_back(it->GetTx()); + } + RemoveStaged(setAllRemoves); } } @@ -161,10 +476,10 @@ void CTxMemPool::removeCoinbaseSpends(const CCoinsViewCache *pcoins, unsigned in // Remove transactions spending a coinbase which are now immature LOCK(cs); list<CTransaction> transactionsToRemove; - for (std::map<uint256, CTxMemPoolEntry>::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { - const CTransaction& tx = it->second.GetTx(); + for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { + const CTransaction& tx = it->GetTx(); BOOST_FOREACH(const CTxIn& txin, tx.vin) { - std::map<uint256, CTxMemPoolEntry>::const_iterator it2 = mapTx.find(txin.prevout.hash); + indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) continue; const CCoins *coins = pcoins->AccessCoins(txin.prevout.hash); @@ -209,8 +524,10 @@ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned i BOOST_FOREACH(const CTransaction& tx, vtx) { uint256 hash = tx.GetHash(); - if (mapTx.count(hash)) - entries.push_back(mapTx[hash]); + + indexed_transaction_set::iterator i = mapTx.find(hash); + if (i != mapTx.end()) + entries.push_back(*i); } BOOST_FOREACH(const CTransaction& tx, vtx) { @@ -226,6 +543,7 @@ void CTxMemPool::removeForBlock(const std::vector<CTransaction>& vtx, unsigned i void CTxMemPool::clear() { LOCK(cs); + mapLinks.clear(); mapTx.clear(); mapNextTx.clear(); totalTxSize = 0; @@ -247,19 +565,25 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const LOCK(cs); list<const CTxMemPoolEntry*> waitingOnDependants; - for (std::map<uint256, CTxMemPoolEntry>::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { + for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { unsigned int i = 0; - checkTotal += it->second.GetTxSize(); - innerUsage += it->second.DynamicMemoryUsage(); - const CTransaction& tx = it->second.GetTx(); + checkTotal += it->GetTxSize(); + innerUsage += it->DynamicMemoryUsage(); + const CTransaction& tx = it->GetTx(); + txlinksMap::const_iterator linksiter = mapLinks.find(it); + assert(linksiter != mapLinks.end()); + const TxLinks &links = linksiter->second; + innerUsage += memusage::DynamicUsage(links.parents) + memusage::DynamicUsage(links.children); bool fDependsWait = false; + setEntries setParentCheck; BOOST_FOREACH(const CTxIn &txin, tx.vin) { // Check that every mempool transaction's inputs refer to available coins, or other mempool tx's. - std::map<uint256, CTxMemPoolEntry>::const_iterator it2 = mapTx.find(txin.prevout.hash); + indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); if (it2 != mapTx.end()) { - const CTransaction& tx2 = it2->second.GetTx(); + const CTransaction& tx2 = it2->GetTx(); assert(tx2.vout.size() > txin.prevout.n && !tx2.vout[txin.prevout.n].IsNull()); fDependsWait = true; + setParentCheck.insert(it2); } else { const CCoins* coins = pcoins->AccessCoins(txin.prevout.hash); assert(coins && coins->IsAvailable(txin.prevout.n)); @@ -271,8 +595,35 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const assert(it3->second.n == i); i++; } + assert(setParentCheck == GetMemPoolParents(it)); + // Check children against mapNextTx + CTxMemPool::setEntries setChildrenCheck; + std::map<COutPoint, CInPoint>::const_iterator iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0)); + int64_t childSizes = 0; + CAmount childFees = 0; + for (; iter != mapNextTx.end() && iter->first.hash == it->GetTx().GetHash(); ++iter) { + txiter childit = mapTx.find(iter->second.ptx->GetHash()); + assert(childit != mapTx.end()); // mapNextTx points to in-mempool transactions + if (setChildrenCheck.insert(childit).second) { + childSizes += childit->GetTxSize(); + childFees += childit->GetFee(); + } + } + assert(setChildrenCheck == GetMemPoolChildren(it)); + // Also check to make sure size/fees is greater than sum with immediate children. + // just a sanity check, not definitive that this calc is correct... + // also check that the size is less than the size of the entire mempool. + if (!it->IsDirty()) { + assert(it->GetSizeWithDescendants() >= childSizes + it->GetTxSize()); + assert(it->GetFeesWithDescendants() >= childFees + it->GetFee()); + } else { + assert(it->GetSizeWithDescendants() == it->GetTxSize()); + assert(it->GetFeesWithDescendants() == it->GetFee()); + } + assert(it->GetFeesWithDescendants() >= 0); + if (fDependsWait) - waitingOnDependants.push_back(&it->second); + waitingOnDependants.push_back(&(*it)); else { CValidationState state; assert(CheckInputs(tx, state, mempoolDuplicate, false, 0, false, NULL)); @@ -296,8 +647,8 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const } for (std::map<COutPoint, CInPoint>::const_iterator it = mapNextTx.begin(); it != mapNextTx.end(); it++) { uint256 hash = it->second.ptx->GetHash(); - map<uint256, CTxMemPoolEntry>::const_iterator it2 = mapTx.find(hash); - const CTransaction& tx = it2->second.GetTx(); + indexed_transaction_set::const_iterator it2 = mapTx.find(hash); + const CTransaction& tx = it2->GetTx(); assert(it2 != mapTx.end()); assert(&tx == it->second.ptx); assert(tx.vin.size() > it->second.n); @@ -314,16 +665,16 @@ void CTxMemPool::queryHashes(vector<uint256>& vtxid) LOCK(cs); vtxid.reserve(mapTx.size()); - for (map<uint256, CTxMemPoolEntry>::iterator mi = mapTx.begin(); mi != mapTx.end(); ++mi) - vtxid.push_back((*mi).first); + for (indexed_transaction_set::iterator mi = mapTx.begin(); mi != mapTx.end(); ++mi) + vtxid.push_back(mi->GetTx().GetHash()); } bool CTxMemPool::lookup(uint256 hash, CTransaction& result) const { LOCK(cs); - map<uint256, CTxMemPoolEntry>::const_iterator i = mapTx.find(hash); + indexed_transaction_set::const_iterator i = mapTx.find(hash); if (i == mapTx.end()) return false; - result = i->second.GetTx(); + result = i->GetTx(); return true; } @@ -429,5 +780,60 @@ bool CCoinsViewMemPool::HaveCoins(const uint256 &txid) const { size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); - return memusage::DynamicUsage(mapTx) + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + cachedInnerUsage; + // Estimate the overhead of mapTx to be 9 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 9 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + cachedInnerUsage; +} + +void CTxMemPool::RemoveStaged(setEntries &stage) { + AssertLockHeld(cs); + UpdateForRemoveFromMempool(stage); + BOOST_FOREACH(const txiter& it, stage) { + removeUnchecked(it); + } +} + +bool CTxMemPool::addUnchecked(const uint256&hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate) +{ + LOCK(cs); + setEntries setAncestors; + uint64_t nNoLimit = std::numeric_limits<uint64_t>::max(); + std::string dummy; + CalculateMemPoolAncestors(entry, setAncestors, nNoLimit, nNoLimit, nNoLimit, nNoLimit, dummy); + return addUnchecked(hash, entry, setAncestors, fCurrentEstimate); +} + +void CTxMemPool::UpdateChild(txiter entry, txiter child, bool add) +{ + setEntries s; + if (add && mapLinks[entry].children.insert(child).second) { + cachedInnerUsage += memusage::IncrementalDynamicUsage(s); + } else if (!add && mapLinks[entry].children.erase(child)) { + cachedInnerUsage -= memusage::IncrementalDynamicUsage(s); + } +} + +void CTxMemPool::UpdateParent(txiter entry, txiter parent, bool add) +{ + setEntries s; + if (add && mapLinks[entry].parents.insert(parent).second) { + cachedInnerUsage += memusage::IncrementalDynamicUsage(s); + } else if (!add && mapLinks[entry].parents.erase(parent)) { + cachedInnerUsage -= memusage::IncrementalDynamicUsage(s); + } +} + +const CTxMemPool::setEntries & CTxMemPool::GetMemPoolParents(txiter entry) const +{ + assert (entry != mapTx.end()); + txlinksMap::const_iterator it = mapLinks.find(entry); + assert(it != mapLinks.end()); + return it->second.parents; +} + +const CTxMemPool::setEntries & CTxMemPool::GetMemPoolChildren(txiter entry) const +{ + assert (entry != mapTx.end()); + txlinksMap::const_iterator it = mapLinks.find(entry); + assert(it != mapLinks.end()); + return it->second.children; } diff --git a/src/txmempool.h b/src/txmempool.h index ea36ce1ad5..c0eef0dd22 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -7,12 +7,17 @@ #define BITCOIN_TXMEMPOOL_H #include <list> +#include <set> #include "amount.h" #include "coins.h" #include "primitives/transaction.h" #include "sync.h" +#undef foreach +#include "boost/multi_index_container.hpp" +#include "boost/multi_index/ordered_index.hpp" + class CAutoFile; inline double AllowFreeThreshold() @@ -30,9 +35,25 @@ inline bool AllowFree(double dPriority) /** Fake height value used in CCoins to signify they are only in the memory pool (since 0.8) */ static const unsigned int MEMPOOL_HEIGHT = 0x7FFFFFFF; -/** - * CTxMemPool stores these: +class CTxMemPool; + +/** \class CTxMemPoolEntry + * + * CTxMemPoolEntry stores data about the correponding transaction, as well + * as data about all in-mempool transactions that depend on the transaction + * ("descendant" transactions). + * + * When a new entry is added to the mempool, we update the descendant state + * (nCountWithDescendants, nSizeWithDescendants, and nFeesWithDescendants) for + * all ancestors of the newly added transaction. + * + * If updating the descendant state is skipped, we can mark the entry as + * "dirty", and set nSizeWithDescendants/nFeesWithDescendants to equal nTxSize/ + * nTxFee. (This can potentially happen during a reorg, where we limit the + * amount of work we're willing to do to avoid consuming too much CPU.) + * */ + class CTxMemPoolEntry { private: @@ -46,10 +67,18 @@ private: unsigned int nHeight; //! Chain height when entering the mempool bool hadNoDependencies; //! Not dependent on any other txs when it entered the mempool + // Information about descendants of this transaction that are in the + // mempool; if we remove this transaction we must remove all of these + // descendants as well. if nCountWithDescendants is 0, treat this entry as + // dirty, and nSizeWithDescendants and nFeesWithDescendants will not be + // correct. + uint64_t nCountWithDescendants; //! number of descendant transactions + uint64_t nSizeWithDescendants; //! ... and size + CAmount nFeesWithDescendants; //! ... and total fees (all including us) + public: CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, int64_t _nTime, double _dPriority, unsigned int _nHeight, bool poolHasNoInputsOf = false); - CTxMemPoolEntry(); CTxMemPoolEntry(const CTxMemPoolEntry& other); const CTransaction& GetTx() const { return this->tx; } @@ -60,6 +89,98 @@ public: unsigned int GetHeight() const { return nHeight; } bool WasClearAtEntry() const { return hadNoDependencies; } size_t DynamicMemoryUsage() const { return nUsageSize; } + + // Adjusts the descendant state, if this entry is not dirty. + void UpdateState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); + + /** We can set the entry to be dirty if doing the full calculation of in- + * mempool descendants will be too expensive, which can potentially happen + * when re-adding transactions from a block back to the mempool. + */ + void SetDirty(); + bool IsDirty() const { return nCountWithDescendants == 0; } + + uint64_t GetCountWithDescendants() const { return nCountWithDescendants; } + uint64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } + CAmount GetFeesWithDescendants() const { return nFeesWithDescendants; } +}; + +// Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. +struct update_descendant_state +{ + update_descendant_state(int64_t _modifySize, CAmount _modifyFee, int64_t _modifyCount) : + modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount) + {} + + void operator() (CTxMemPoolEntry &e) + { e.UpdateState(modifySize, modifyFee, modifyCount); } + + private: + int64_t modifySize; + CAmount modifyFee; + int64_t modifyCount; +}; + +struct set_dirty +{ + void operator() (CTxMemPoolEntry &e) + { e.SetDirty(); } +}; + +// extracts a TxMemPoolEntry's transaction hash +struct mempoolentry_txid +{ + typedef uint256 result_type; + result_type operator() (const CTxMemPoolEntry &entry) const + { + return entry.GetTx().GetHash(); + } +}; + +/** \class CompareTxMemPoolEntryByFee + * + * Sort an entry by max(feerate of entry's tx, feerate with all descendants). + */ +class CompareTxMemPoolEntryByFee +{ +public: + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + { + bool fUseADescendants = UseDescendantFeeRate(a); + bool fUseBDescendants = UseDescendantFeeRate(b); + + double aFees = fUseADescendants ? a.GetFeesWithDescendants() : a.GetFee(); + double aSize = fUseADescendants ? a.GetSizeWithDescendants() : a.GetTxSize(); + + double bFees = fUseBDescendants ? b.GetFeesWithDescendants() : b.GetFee(); + double bSize = fUseBDescendants ? b.GetSizeWithDescendants() : b.GetTxSize(); + + // Avoid division by rewriting (a/b > c/d) as (a*d > c*b). + double f1 = aFees * bSize; + double f2 = aSize * bFees; + + if (f1 == f2) { + return a.GetTime() < b.GetTime(); + } + return f1 > f2; + } + + // Calculate which feerate to use for an entry (avoiding division). + bool UseDescendantFeeRate(const CTxMemPoolEntry &a) + { + double f1 = (double)a.GetFee() * a.GetSizeWithDescendants(); + double f2 = (double)a.GetFeesWithDescendants() * a.GetTxSize(); + return f2 > f1; + } +}; + +class CompareTxMemPoolEntryByEntryTime +{ +public: + bool operator()(const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) + { + return a.GetTime() < b.GetTime(); + } }; class CBlockPolicyEstimator; @@ -87,6 +208,71 @@ public: * are added to the pool: if a new transaction double-spends * an input of a transaction in the pool, it is dropped, * as are non-standard transactions. + * + * CTxMemPool::mapTx, and CTxMemPoolEntry bookkeeping: + * + * mapTx is a boost::multi_index that sorts the mempool on 2 criteria: + * - transaction hash + * - feerate [we use max(feerate of tx, feerate of tx with all descendants)] + * + * Note: the term "descendant" refers to in-mempool transactions that depend on + * this one, while "ancestor" refers to in-mempool transactions that a given + * transaction depends on. + * + * In order for the feerate sort to remain correct, we must update transactions + * in the mempool when new descendants arrive. To facilitate this, we track + * the set of in-mempool direct parents and direct children in mapLinks. Within + * each CTxMemPoolEntry, we track the size and fees of all descendants. + * + * Usually when a new transaction is added to the mempool, it has no in-mempool + * children (because any such children would be an orphan). So in + * addUnchecked(), we: + * - update a new entry's setMemPoolParents to include all in-mempool parents + * - update the new entry's direct parents to include the new tx as a child + * - update all ancestors of the transaction to include the new tx's size/fee + * + * When a transaction is removed from the mempool, we must: + * - update all in-mempool parents to not track the tx in setMemPoolChildren + * - update all ancestors to not include the tx's size/fees in descendant state + * - update all in-mempool children to not include it as a parent + * + * These happen in UpdateForRemoveFromMempool(). (Note that when removing a + * transaction along with its descendants, we must calculate that set of + * transactions to be removed before doing the removal, or else the mempool can + * be in an inconsistent state where it's impossible to walk the ancestors of + * a transaction.) + * + * In the event of a reorg, the assumption that a newly added tx has no + * in-mempool children is false. In particular, the mempool is in an + * inconsistent state while new transactions are being added, because there may + * be descendant transactions of a tx coming from a disconnected block that are + * unreachable from just looking at transactions in the mempool (the linking + * transactions may also be in the disconnected block, waiting to be added). + * Because of this, there's not much benefit in trying to search for in-mempool + * children in addUnchecked(). Instead, in the special case of transactions + * being added from a disconnected block, we require the caller to clean up the + * state, to account for in-mempool, out-of-block descendants for all the + * in-block transactions by calling UpdateTransactionsFromBlock(). Note that + * until this is called, the mempool state is not consistent, and in particular + * mapLinks may not be correct (and therefore functions like + * CalculateMemPoolAncestors() and CalculateDescendants() that rely + * on them to walk the mempool are not generally safe to use). + * + * Computational limits: + * + * Updating all in-mempool ancestors of a newly added transaction can be slow, + * if no bound exists on how many in-mempool ancestors there may be. + * CalculateMemPoolAncestors() takes configurable limits that are designed to + * prevent these calculations from being too CPU intensive. + * + * Adding transactions from a disconnected block can be very time consuming, + * because we don't have a way to limit the number of in-mempool descendants. + * To bound CPU processing, we limit the amount of work we're willing to do + * to properly update the descendant information for a tx being added from + * a disconnected block. If we would exceed the limit, then we instead mark + * the entry as "dirty", and set the feerate for sorting purposes to be equal + * the feerate of the transaction without any descendants. + * */ class CTxMemPool { @@ -99,8 +285,46 @@ private: uint64_t cachedInnerUsage; //! sum of dynamic memory usage of all the map elements (NOT the maps themselves) public: + typedef boost::multi_index_container< + CTxMemPoolEntry, + boost::multi_index::indexed_by< + // sorted by txid + boost::multi_index::ordered_unique<mempoolentry_txid>, + // sorted by fee rate + boost::multi_index::ordered_non_unique< + boost::multi_index::identity<CTxMemPoolEntry>, + CompareTxMemPoolEntryByFee + > + > + > indexed_transaction_set; + mutable CCriticalSection cs; - std::map<uint256, CTxMemPoolEntry> mapTx; + indexed_transaction_set mapTx; + typedef indexed_transaction_set::nth_index<0>::type::iterator txiter; + struct CompareIteratorByHash { + bool operator()(const txiter &a, const txiter &b) const { + return a->GetTx().GetHash() < b->GetTx().GetHash(); + } + }; + typedef std::set<txiter, CompareIteratorByHash> setEntries; + +private: + typedef std::map<txiter, setEntries, CompareIteratorByHash> cacheMap; + + struct TxLinks { + setEntries parents; + setEntries children; + }; + + typedef std::map<txiter, TxLinks, CompareIteratorByHash> txlinksMap; + txlinksMap mapLinks; + + const setEntries & GetMemPoolParents(txiter entry) const; + const setEntries & GetMemPoolChildren(txiter entry) const; + void UpdateParent(txiter entry, txiter parent, bool add); + void UpdateChild(txiter entry, txiter child, bool add); + +public: std::map<COutPoint, CInPoint> mapNextTx; std::map<uint256, std::pair<double, CAmount> > mapDeltas; @@ -116,7 +340,13 @@ public: void check(const CCoinsViewCache *pcoins) const; void setSanityCheck(bool _fSanityCheck) { fSanityCheck = _fSanityCheck; } + // addUnchecked must updated state for all ancestors of a given transaction, + // to track size/count of descendant transactions. First version of + // addUnchecked can be used to have it call CalculateMemPoolAncestors(), and + // then invoke the second version. bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); + bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool fCurrentEstimate = true); + void remove(const CTransaction &tx, std::list<CTransaction>& removed, bool fRecursive = false); void removeCoinbaseSpends(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight); void removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed); @@ -138,6 +368,35 @@ public: void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta); void ClearPrioritisation(const uint256 hash); +public: + /** Remove a set of transactions from the mempool. + * If a transaction is in this set, then all in-mempool descendants must + * also be in the set.*/ + void RemoveStaged(setEntries &stage); + + /** When adding transactions from a disconnected block back to the mempool, + * new mempool entries may have children in the mempool (which is generally + * not the case when otherwise adding transactions). + * UpdateTransactionsFromBlock() will find child transactions and update the + * descendant state for each transaction in hashesToUpdate (excluding any + * child transactions present in hashesToUpdate, which are already accounted + * for). Note: hashesToUpdate should be the set of transactions from the + * disconnected block that have been accepted back into the mempool. + */ + void UpdateTransactionsFromBlock(const std::vector<uint256> &hashesToUpdate); + + /** Try to calculate all in-mempool ancestors of entry. + * (these are all calculated including the tx itself) + * limitAncestorCount = max number of ancestors + * limitAncestorSize = max size of ancestors + * limitDescendantCount = max number of descendants any ancestor can have + * limitDescendantSize = max size of descendants any ancestor can have + * errString = populated with error reason if any limits are hit + * fSearchForParents = whether to search a tx's vin for in-mempool parents, or + * look up parents from mapLinks. Must be true for entries not in the mempool + */ + bool CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents = true); + unsigned long size() { LOCK(cs); @@ -169,6 +428,48 @@ public: bool ReadFeeEstimates(CAutoFile& filein); size_t DynamicMemoryUsage() const; + +private: + /** UpdateForDescendants is used by UpdateTransactionsFromBlock to update + * the descendants for a single transaction that has been added to the + * mempool but may have child transactions in the mempool, eg during a + * chain reorg. setExclude is the set of descendant transactions in the + * mempool that must not be accounted for (because any descendants in + * setExclude were added to the mempool after the transaction being + * updated and hence their state is already reflected in the parent + * state). + * + * If updating an entry requires looking at more than maxDescendantsToVisit + * transactions, outside of the ones in setExclude, then give up. + * + * cachedDescendants will be updated with the descendants of the transaction + * being updated, so that future invocations don't need to walk the + * same transaction again, if encountered in another transaction chain. + */ + bool UpdateForDescendants(txiter updateIt, + int maxDescendantsToVisit, + cacheMap &cachedDescendants, + const std::set<uint256> &setExclude); + /** Update ancestors of hash to add/remove it as a descendant transaction. */ + void UpdateAncestorsOf(bool add, txiter hash, setEntries &setAncestors); + /** For each transaction being removed, update ancestors and any direct children. */ + void UpdateForRemoveFromMempool(const setEntries &entriesToRemove); + /** Sever link between specified transaction and direct children. */ + void UpdateChildrenForRemoval(txiter entry); + /** Populate setDescendants with all in-mempool descendants of hash. + * Assumes that setDescendants includes all in-mempool descendants of anything + * already in it. */ + void CalculateDescendants(txiter it, setEntries &setDescendants); + + /** Before calling removeUnchecked for a given transaction, + * UpdateForRemoveFromMempool must be called on the entire (dependent) set + * of transactions being removed at the same time. We use each + * CTxMemPoolEntry's setMemPoolParents in order to walk ancestors of a + * given transaction that is removed, so we can't remove intermediate + * transactions in a chain before we've updated all the state for the + * removal. + */ + void removeUnchecked(txiter entry); }; /** diff --git a/src/ui_interface.h b/src/ui_interface.h index 32a92a4b81..e402479933 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -95,6 +95,9 @@ public: /** New block has been accepted */ boost::signals2::signal<void (const uint256& hash)> NotifyBlockTip; + + /** Banlist did change. */ + boost::signals2::signal<void (void)> BannedListChanged; }; extern CClientUIInterface uiInterface; diff --git a/src/util.cpp b/src/util.cpp index a7ec740de8..f50d25e17a 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -794,6 +794,18 @@ void SetupEnvironment() boost::filesystem::path::imbue(loc); } +bool SetupNetworking() +{ +#ifdef WIN32 + // Initialize Windows Sockets + WSADATA wsadata; + int ret = WSAStartup(MAKEWORD(2,2), &wsadata); + if (ret != NO_ERROR || LOBYTE(wsadata.wVersion ) != 2 || HIBYTE(wsadata.wVersion) != 2) + return false; +#endif + return true; +} + void SetThreadPriority(int nPriority) { #ifdef WIN32 diff --git a/src/util.h b/src/util.h index afc9a378bb..0b2dc01ac6 100644 --- a/src/util.h +++ b/src/util.h @@ -59,6 +59,7 @@ inline std::string _(const char* psz) } void SetupEnvironment(); +bool SetupNetworking(); /** Return true if log accepts specified category */ bool LogAcceptCategory(const char* category); diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index 1f7a2cae2c..76c22f7353 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -14,17 +14,20 @@ using namespace std; -string SanitizeString(const string& str) +static const string CHARS_ALPHA_NUM = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + +static const string SAFE_CHARS[] = +{ + CHARS_ALPHA_NUM + " .,;_/:?@()", // SAFE_CHARS_DEFAULT + CHARS_ALPHA_NUM + " .,;_?@" // SAFE_CHARS_UA_COMMENT +}; + +string SanitizeString(const string& str, int rule) { - /** - * safeChars chosen to allow simple messages/URLs/email addresses, but avoid anything - * even possibly remotely dangerous like & or > - */ - static string safeChars("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890 .,;_/:?@()"); string strResult; for (std::string::size_type i = 0; i < str.size(); i++) { - if (safeChars.find(str[i]) != std::string::npos) + if (SAFE_CHARS[rule].find(str[i]) != std::string::npos) strResult.push_back(str[i]); } return strResult; diff --git a/src/utilstrencodings.h b/src/utilstrencodings.h index dcd56751f2..ce93e83497 100644 --- a/src/utilstrencodings.h +++ b/src/utilstrencodings.h @@ -22,7 +22,21 @@ /** This is needed because the foreach macro can't get over the comma in pair<t1, t2> */ #define PAIRTYPE(t1, t2) std::pair<t1, t2> -std::string SanitizeString(const std::string& str); +/** Used by SanitizeString() */ +enum SafeChars +{ + SAFE_CHARS_DEFAULT, //!< The full set of allowed chars + SAFE_CHARS_UA_COMMENT //!< BIP-0014 subset +}; + +/** +* Remove unsafe chars. Safe chars chosen to allow simple messages/URLs/email +* addresses, but avoid anything even possibly remotely dangerous like & or > +* @param[in] str The string to sanitize +* @param[in] rule The set of safe chars to choose (default: least restrictive) +* @return A new string without unsafe chars +*/ +std::string SanitizeString(const std::string& str, int rule = SAFE_CHARS_DEFAULT); std::vector<unsigned char> ParseHex(const char* psz); std::vector<unsigned char> ParseHex(const std::string& str); signed char HexDigit(char c); diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index d365f03008..81f3b775f4 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -13,6 +13,7 @@ CMainSignals& GetMainSignals() } void RegisterValidationInterface(CValidationInterface* pwalletIn) { + g_signals.UpdatedBlockTip.connect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1)); g_signals.SyncTransaction.connect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2)); g_signals.UpdatedTransaction.connect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); @@ -32,6 +33,7 @@ void UnregisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.UpdatedTransaction.disconnect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); g_signals.SyncTransaction.disconnect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2)); + g_signals.UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1)); } void UnregisterAllValidationInterfaces() { @@ -43,6 +45,7 @@ void UnregisterAllValidationInterfaces() { g_signals.SetBestChain.disconnect_all_slots(); g_signals.UpdatedTransaction.disconnect_all_slots(); g_signals.SyncTransaction.disconnect_all_slots(); + g_signals.UpdatedBlockTip.disconnect_all_slots(); } void SyncWithWallets(const CTransaction &tx, const CBlock *pblock) { diff --git a/src/validationinterface.h b/src/validationinterface.h index fb0ce0bdaa..6f95ad74eb 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -30,6 +30,7 @@ void SyncWithWallets(const CTransaction& tx, const CBlock* pblock = NULL); class CValidationInterface { protected: + virtual void UpdatedBlockTip(const uint256 &newHashTip) {} virtual void SyncTransaction(const CTransaction &tx, const CBlock *pblock) {} virtual void SetBestChain(const CBlockLocator &locator) {} virtual void UpdatedTransaction(const uint256 &hash) {} @@ -44,6 +45,8 @@ protected: }; struct CMainSignals { + /** Notifies listeners of updated block chain tip */ + boost::signals2::signal<void (const uint256 &)> UpdatedBlockTip; /** Notifies listeners of updated transaction data (transaction, and optionally the block it is found in. */ boost::signals2::signal<void (const CTransaction &, const CBlock *)> SyncTransaction; /** Notifies listeners of an updated transaction without new data (for now: a coinbase potentially becoming visible). */ diff --git a/src/version.h b/src/version.h index 38b3d2e734..6cdddf9255 100644 --- a/src/version.h +++ b/src/version.h @@ -9,7 +9,7 @@ * network protocol versioning */ -static const int PROTOCOL_VERSION = 70002; +static const int PROTOCOL_VERSION = 70011; //! initial proto version, to be increased after version/verack negotiation static const int INIT_PROTO_VERSION = 209; @@ -34,4 +34,7 @@ static const int BIP0031_VERSION = 60000; //! "mempool" command, enhanced "getdata" behavior starts with this version static const int MEMPOOL_GD_VERSION = 60002; +//! "filter*" commands are disabled without NODE_BLOOM after and including this version +static const int NO_BLOOM_VERSION = 70011; + #endif // BITCOIN_VERSION_H diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 8d557979c0..7e22faac37 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -97,8 +97,6 @@ UniValue importprivkey(const UniValue& params, bool fHelp) + HelpExampleRpc("importprivkey", "\"mykey\", \"testing\", false") ); - if (fPruneMode) - throw JSONRPCError(RPC_WALLET_ERROR, "Importing keys is disabled in pruned mode"); LOCK2(cs_main, pwalletMain->cs_wallet); @@ -114,6 +112,9 @@ UniValue importprivkey(const UniValue& params, bool fHelp) if (params.size() > 2) fRescan = params[2].get_bool(); + if (fRescan && fPruneMode) + throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + CBitcoinSecret vchSecret; bool fGood = vchSecret.SetString(strSecret); @@ -201,8 +202,6 @@ UniValue importaddress(const UniValue& params, bool fHelp) + HelpExampleRpc("importaddress", "\"myscript\", \"testing\", false") ); - if (fPruneMode) - throw JSONRPCError(RPC_WALLET_ERROR, "Importing addresses is disabled in pruned mode"); string strLabel = ""; if (params.size() > 1) @@ -213,6 +212,9 @@ UniValue importaddress(const UniValue& params, bool fHelp) if (params.size() > 2) fRescan = params[2].get_bool(); + if (fRescan && fPruneMode) + throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + // Whether to import a p2sh version, too bool fP2SH = false; if (params.size() > 3) @@ -264,8 +266,6 @@ UniValue importpubkey(const UniValue& params, bool fHelp) + HelpExampleRpc("importpubkey", "\"mypubkey\", \"testing\", false") ); - if (fPruneMode) - throw JSONRPCError(RPC_WALLET_ERROR, "Importing public keys is disabled in pruned mode"); string strLabel = ""; if (params.size() > 1) @@ -276,6 +276,9 @@ UniValue importpubkey(const UniValue& params, bool fHelp) if (params.size() > 2) fRescan = params[2].get_bool(); + if (fRescan && fPruneMode) + throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + if (!IsHex(params[0].get_str())) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey must be a hex string"); std::vector<unsigned char> data(ParseHex(params[0].get_str())); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index c3b1172201..39eff3af30 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -702,9 +702,8 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFromLoadWallet, CWalletD wtx.hashBlock = wtxIn.hashBlock; fUpdated = true; } - if (wtxIn.nIndex != -1 && (wtxIn.vMerkleBranch != wtx.vMerkleBranch || wtxIn.nIndex != wtx.nIndex)) + if (wtxIn.nIndex != -1 && (wtxIn.nIndex != wtx.nIndex)) { - wtx.vMerkleBranch = wtxIn.vMerkleBranch; wtx.nIndex = wtxIn.nIndex; fUpdated = true; } @@ -2812,15 +2811,11 @@ int CMerkleTx::SetMerkleBranch(const CBlock& block) break; if (nIndex == (int)block.vtx.size()) { - vMerkleBranch.clear(); nIndex = -1; LogPrintf("ERROR: SetMerkleBranch(): couldn't find tx in block\n"); return 0; } - // Fill in merkle branch - vMerkleBranch = block.GetMerkleBranch(nIndex); - // Is the tx in a block that's in the main chain BlockMap::iterator mi = mapBlockIndex.find(hashBlock); if (mi == mapBlockIndex.end()) @@ -2846,14 +2841,6 @@ int CMerkleTx::GetDepthInMainChainINTERNAL(const CBlockIndex* &pindexRet) const if (!pindex || !chainActive.Contains(pindex)) return 0; - // Make sure the merkle branch connects to this block - if (!fMerkleVerified) - { - if (CBlock::CheckMerkleBranch(GetHash(), vMerkleBranch, nIndex) != pindex->hashMerkleRoot) - return 0; - fMerkleVerified = true; - } - pindexRet = pindex; return chainActive.Height() - pindex->nHeight + 1; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index bd30b67b09..34e98cfb81 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -151,13 +151,8 @@ private: public: uint256 hashBlock; - std::vector<uint256> vMerkleBranch; int nIndex; - // memory only - mutable bool fMerkleVerified; - - CMerkleTx() { Init(); @@ -172,13 +167,13 @@ public: { hashBlock = uint256(); nIndex = -1; - fMerkleVerified = false; } ADD_SERIALIZE_METHODS; template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + std::vector<uint256> vMerkleBranch; // For compatibility with older versions. READWRITE(*(CTransaction*)this); nVersion = this->nVersion; READWRITE(hashBlock); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index c1eb184581..0624e442d1 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -131,12 +131,14 @@ bool CWalletDB::EraseWatchOnly(const CScript &dest) bool CWalletDB::WriteBestBlock(const CBlockLocator& locator) { nWalletDBUpdated++; - return Write(std::string("bestblock"), locator); + Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan + return Write(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::ReadBestBlock(CBlockLocator& locator) { - return Read(std::string("bestblock"), locator); + if (Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; + return Read(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::WriteOrderPosNext(int64_t nOrderPosNext) diff --git a/src/zmq/zmqabstractnotifier.cpp b/src/zmq/zmqabstractnotifier.cpp new file mode 100644 index 0000000000..744ec59234 --- /dev/null +++ b/src/zmq/zmqabstractnotifier.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "zmqabstractnotifier.h" +#include "util.h" + + +CZMQAbstractNotifier::~CZMQAbstractNotifier() +{ + assert(!psocket); +} + +bool CZMQAbstractNotifier::NotifyBlock(const uint256 &/*hash*/) +{ + return true; +} + +bool CZMQAbstractNotifier::NotifyTransaction(const CTransaction &/*transaction*/) +{ + return true; +} diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h new file mode 100644 index 0000000000..626d1ddf92 --- /dev/null +++ b/src/zmq/zmqabstractnotifier.h @@ -0,0 +1,42 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H +#define BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H + +#include "zmqconfig.h" + +class CZMQAbstractNotifier; +typedef CZMQAbstractNotifier* (*CZMQNotifierFactory)(); + +class CZMQAbstractNotifier +{ +public: + CZMQAbstractNotifier() : psocket(0) { } + virtual ~CZMQAbstractNotifier(); + + template <typename T> + static CZMQAbstractNotifier* Create() + { + return new T(); + } + + std::string GetType() const { return type; } + void SetType(const std::string &t) { type = t; } + std::string GetAddress() const { return address; } + void SetAddress(const std::string &a) { address = a; } + + virtual bool Initialize(void *pcontext) = 0; + virtual void Shutdown() = 0; + + virtual bool NotifyBlock(const uint256 &hash); + virtual bool NotifyTransaction(const CTransaction &transaction); + +protected: + void *psocket; + std::string type; + std::string address; +}; + +#endif // BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H diff --git a/src/zmq/zmqconfig.h b/src/zmq/zmqconfig.h new file mode 100644 index 0000000000..6057f5d1a0 --- /dev/null +++ b/src/zmq/zmqconfig.h @@ -0,0 +1,24 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ZMQ_ZMQCONFIG_H +#define BITCOIN_ZMQ_ZMQCONFIG_H + +#if defined(HAVE_CONFIG_H) +#include "config/bitcoin-config.h" +#endif + +#include <stdarg.h> +#include <string> + +#if ENABLE_ZMQ +#include <zmq.h> +#endif + +#include "primitives/block.h" +#include "primitives/transaction.h" + +void zmqError(const char *str); + +#endif // BITCOIN_ZMQ_ZMQCONFIG_H diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp new file mode 100644 index 0000000000..71ccb59a4a --- /dev/null +++ b/src/zmq/zmqnotificationinterface.cpp @@ -0,0 +1,155 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "zmqnotificationinterface.h" +#include "zmqpublishnotifier.h" + +#include "version.h" +#include "main.h" +#include "streams.h" +#include "util.h" + +void zmqError(const char *str) +{ + LogPrint("zmq", "Error: %s, errno=%s\n", str, zmq_strerror(errno)); +} + +CZMQNotificationInterface::CZMQNotificationInterface() : pcontext(NULL) +{ +} + +CZMQNotificationInterface::~CZMQNotificationInterface() +{ + // ensure Shutdown if Initialize is called + assert(!pcontext); + + for (std::list<CZMQAbstractNotifier*>::iterator i=notifiers.begin(); i!=notifiers.end(); ++i) + { + delete *i; + } +} + +CZMQNotificationInterface* CZMQNotificationInterface::CreateWithArguments(const std::map<std::string, std::string> &args) +{ + CZMQNotificationInterface* notificationInterface = NULL; + std::map<std::string, CZMQNotifierFactory> factories; + std::list<CZMQAbstractNotifier*> notifiers; + + factories["pubhashblock"] = CZMQAbstractNotifier::Create<CZMQPublishHashBlockNotifier>; + factories["pubhashtx"] = CZMQAbstractNotifier::Create<CZMQPublishHashTransactionNotifier>; + factories["pubrawblock"] = CZMQAbstractNotifier::Create<CZMQPublishRawBlockNotifier>; + factories["pubrawtx"] = CZMQAbstractNotifier::Create<CZMQPublishRawTransactionNotifier>; + + for (std::map<std::string, CZMQNotifierFactory>::const_iterator i=factories.begin(); i!=factories.end(); ++i) + { + std::map<std::string, std::string>::const_iterator j = args.find("-zmq" + i->first); + if (j!=args.end()) + { + CZMQNotifierFactory factory = i->second; + std::string address = j->second; + CZMQAbstractNotifier *notifier = factory(); + notifier->SetType(i->first); + notifier->SetAddress(address); + notifiers.push_back(notifier); + } + } + + if (!notifiers.empty()) + { + notificationInterface = new CZMQNotificationInterface(); + notificationInterface->notifiers = notifiers; + } + + return notificationInterface; +} + +// Called at startup to conditionally set up ZMQ socket(s) +bool CZMQNotificationInterface::Initialize() +{ + LogPrint("zmq", "Initialize notification interface\n"); + assert(!pcontext); + + pcontext = zmq_init(1); + + if (!pcontext) + { + zmqError("Unable to initialize context"); + return false; + } + + std::list<CZMQAbstractNotifier*>::iterator i=notifiers.begin(); + for (; i!=notifiers.end(); ++i) + { + CZMQAbstractNotifier *notifier = *i; + if (notifier->Initialize(pcontext)) + { + LogPrint("zmq", " Notifier %s ready (address = %s)\n", notifier->GetType(), notifier->GetAddress()); + } + else + { + LogPrint("zmq", " Notifier %s failed (address = %s)\n", notifier->GetType(), notifier->GetAddress()); + break; + } + } + + if (i!=notifiers.end()) + { + Shutdown(); + return false; + } + + return false; +} + +// Called during shutdown sequence +void CZMQNotificationInterface::Shutdown() +{ + LogPrint("zmq", "Shutdown notification interface\n"); + if (pcontext) + { + for (std::list<CZMQAbstractNotifier*>::iterator i=notifiers.begin(); i!=notifiers.end(); ++i) + { + CZMQAbstractNotifier *notifier = *i; + LogPrint("zmq", " Shutdown notifier %s at %s\n", notifier->GetType(), notifier->GetAddress()); + notifier->Shutdown(); + } + zmq_ctx_destroy(pcontext); + + pcontext = 0; + } +} + +void CZMQNotificationInterface::UpdatedBlockTip(const uint256 &hash) +{ + for (std::list<CZMQAbstractNotifier*>::iterator i = notifiers.begin(); i!=notifiers.end(); ) + { + CZMQAbstractNotifier *notifier = *i; + if (notifier->NotifyBlock(hash)) + { + i++; + } + else + { + notifier->Shutdown(); + i = notifiers.erase(i); + } + } +} + +void CZMQNotificationInterface::SyncTransaction(const CTransaction &tx, const CBlock *pblock) +{ + for (std::list<CZMQAbstractNotifier*>::iterator i = notifiers.begin(); i!=notifiers.end(); ) + { + CZMQAbstractNotifier *notifier = *i; + if (notifier->NotifyTransaction(tx)) + { + i++; + } + else + { + notifier->Shutdown(); + i = notifiers.erase(i); + } + } +} diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h new file mode 100644 index 0000000000..afc0b8d24e --- /dev/null +++ b/src/zmq/zmqnotificationinterface.h @@ -0,0 +1,35 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H +#define BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H + +#include "validationinterface.h" +#include <string> +#include <map> + +class CZMQAbstractNotifier; + +class CZMQNotificationInterface : public CValidationInterface +{ +public: + virtual ~CZMQNotificationInterface(); + + static CZMQNotificationInterface* CreateWithArguments(const std::map<std::string, std::string> &args); + + bool Initialize(); + void Shutdown(); + +protected: // CValidationInterface + void SyncTransaction(const CTransaction &tx, const CBlock *pblock); + void UpdatedBlockTip(const uint256 &newHashTip); + +private: + CZMQNotificationInterface(); + + void *pcontext; + std::list<CZMQAbstractNotifier*> notifiers; +}; + +#endif // BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp new file mode 100644 index 0000000000..0a6d7d0dbc --- /dev/null +++ b/src/zmq/zmqpublishnotifier.cpp @@ -0,0 +1,172 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "zmqpublishnotifier.h" +#include "main.h" +#include "util.h" + +static std::multimap<std::string, CZMQAbstractPublishNotifier*> mapPublishNotifiers; + +// Internal function to send multipart message +static int zmq_send_multipart(void *sock, const void* data, size_t size, ...) +{ + va_list args; + va_start(args, size); + + while (1) + { + zmq_msg_t msg; + + int rc = zmq_msg_init_size(&msg, size); + if (rc != 0) + { + zmqError("Unable to initialize ZMQ msg"); + return -1; + } + + void *buf = zmq_msg_data(&msg); + memcpy(buf, data, size); + + data = va_arg(args, const void*); + + rc = zmq_msg_send(&msg, sock, data ? ZMQ_SNDMORE : 0); + if (rc == -1) + { + zmqError("Unable to send ZMQ msg"); + zmq_msg_close(&msg); + return -1; + } + + zmq_msg_close(&msg); + + if (!data) + break; + + size = va_arg(args, size_t); + } + return 0; +} + +bool CZMQAbstractPublishNotifier::Initialize(void *pcontext) +{ + assert(!psocket); + + // check if address is being used by other publish notifier + std::multimap<std::string, CZMQAbstractPublishNotifier*>::iterator i = mapPublishNotifiers.find(address); + + if (i==mapPublishNotifiers.end()) + { + psocket = zmq_socket(pcontext, ZMQ_PUB); + if (!psocket) + { + zmqError("Failed to create socket"); + return false; + } + + int rc = zmq_bind(psocket, address.c_str()); + if (rc!=0) + { + zmqError("Failed to bind address"); + return false; + } + + // register this notifier for the address, so it can be reused for other publish notifier + mapPublishNotifiers.insert(std::make_pair(address, this)); + return true; + } + else + { + LogPrint("zmq", " Reuse socket for address %s\n", address); + + psocket = i->second->psocket; + mapPublishNotifiers.insert(std::make_pair(address, this)); + + return true; + } +} + +void CZMQAbstractPublishNotifier::Shutdown() +{ + assert(psocket); + + int count = mapPublishNotifiers.count(address); + + // remove this notifier from the list of publishers using this address + typedef std::multimap<std::string, CZMQAbstractPublishNotifier*>::iterator iterator; + std::pair<iterator, iterator> iterpair = mapPublishNotifiers.equal_range(address); + + for (iterator it = iterpair.first; it != iterpair.second; ++it) + { + if (it->second==this) + { + mapPublishNotifiers.erase(it); + break; + } + } + + if (count == 1) + { + LogPrint("zmq", "Close socket at address %s\n", address); + int linger = 0; + zmq_setsockopt(psocket, ZMQ_LINGER, &linger, sizeof(linger)); + zmq_close(psocket); + } + + psocket = 0; +} + +bool CZMQPublishHashBlockNotifier::NotifyBlock(const uint256 &hash) +{ + LogPrint("zmq", "Publish hash block %s\n", hash.GetHex()); + char data[32]; + for (unsigned int i = 0; i < 32; i++) + data[31 - i] = hash.begin()[i]; + int rc = zmq_send_multipart(psocket, "hashblock", 9, data, 32, 0); + return rc == 0; +} + +bool CZMQPublishHashTransactionNotifier::NotifyTransaction(const CTransaction &transaction) +{ + uint256 hash = transaction.GetHash(); + LogPrint("zmq", "Publish hash transaction %s\n", hash.GetHex()); + char data[32]; + for (unsigned int i = 0; i < 32; i++) + data[31 - i] = hash.begin()[i]; + int rc = zmq_send_multipart(psocket, "hashtx", 6, data, 32, 0); + return rc == 0; +} + +bool CZMQPublishRawBlockNotifier::NotifyBlock(const uint256 &hash) +{ + LogPrint("zmq", "Publish raw block %s\n", hash.GetHex()); + + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + { + LOCK(cs_main); + + CBlock block; + CBlockIndex* pblockindex = mapBlockIndex[hash]; + + if(!ReadBlockFromDisk(block, pblockindex)) + { + zmqError("Can't read block from disk"); + return false; + } + + ss << block; + } + + int rc = zmq_send_multipart(psocket, "rawblock", 8, &(*ss.begin()), ss.size(), 0); + return rc == 0; +} + +bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &transaction) +{ + uint256 hash = transaction.GetHash(); + LogPrint("zmq", "Publish raw transaction %s\n", hash.GetHex()); + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << transaction; + int rc = zmq_send_multipart(psocket, "rawtx", 5, &(*ss.begin()), ss.size(), 0); + return rc == 0; +} diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h new file mode 100644 index 0000000000..a0eb26f5e2 --- /dev/null +++ b/src/zmq/zmqpublishnotifier.h @@ -0,0 +1,41 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ZMQ_ZMQPUBLISHNOTIFIER_H +#define BITCOIN_ZMQ_ZMQPUBLISHNOTIFIER_H + +#include "zmqabstractnotifier.h" + +class CZMQAbstractPublishNotifier : public CZMQAbstractNotifier +{ +public: + bool Initialize(void *pcontext); + void Shutdown(); +}; + +class CZMQPublishHashBlockNotifier : public CZMQAbstractPublishNotifier +{ +public: + bool NotifyBlock(const uint256 &hash); +}; + +class CZMQPublishHashTransactionNotifier : public CZMQAbstractPublishNotifier +{ +public: + bool NotifyTransaction(const CTransaction &transaction); +}; + +class CZMQPublishRawBlockNotifier : public CZMQAbstractPublishNotifier +{ +public: + bool NotifyBlock(const uint256 &hash); +}; + +class CZMQPublishRawTransactionNotifier : public CZMQAbstractPublishNotifier +{ +public: + bool NotifyTransaction(const CTransaction &transaction); +}; + +#endif // BITCOIN_ZMQ_ZMQPUBLISHNOTIFIER_H |
