diff options
74 files changed, 2478 insertions, 1202 deletions
diff --git a/configure.ac b/configure.ac index cc96511f97..b94218f3ee 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N) AC_PREREQ([2.60]) define(_CLIENT_VERSION_MAJOR, 0) -define(_CLIENT_VERSION_MINOR, 12) +define(_CLIENT_VERSION_MINOR, 13) define(_CLIENT_VERSION_REVISION, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_IS_RELEASE, false) @@ -816,6 +816,15 @@ else AC_DEFINE_UNQUOTED([ENABLE_ZMQ],[0],[Define to 1 to enable ZMQ functions]) fi + if test "x$use_zmq" = "xyes"; then + dnl Assume libzmq was built for static linking + case $host in + *mingw*) + ZMQ_CFLAGS="$ZMQ_CFLAGS -DZMQ_STATIC" + ;; + esac + fi + BITCOIN_QT_CHECK(AC_CHECK_LIB([protobuf] ,[main],[PROTOBUF_LIBS=-lprotobuf], BITCOIN_QT_FAIL(libprotobuf not found))) if test x$use_qr != xno; then BITCOIN_QT_CHECK([AC_CHECK_LIB([qrencode], [main],[QR_LIBS=-lqrencode], [have_qrencode=no])]) diff --git a/depends/packages/native_biplist.mk b/depends/packages/native_biplist.mk index eb8672d556..3c6e8900f6 100644 --- a/depends/packages/native_biplist.mk +++ b/depends/packages/native_biplist.mk @@ -4,6 +4,11 @@ $(package)_download_path=https://pypi.python.org/packages/source/b/biplist $(package)_file_name=biplist-$($(package)_version).tar.gz $(package)_sha256_hash=b57cadfd26e4754efdf89e9e37de87885f9b5c847b2615688ca04adfaf6ca604 $(package)_install_libdir=$(build_prefix)/lib/python/dist-packages +$(package)_patches=sorted_list.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/sorted_list.patch +endef define $(package)_build_cmds python setup.py build diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index ac43ef4a2e..ba2a05248c 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,6 +1,4 @@ -packages:=boost openssl libevent -darwin_packages:=zeromq -linux_packages:=zeromq +packages:=boost openssl libevent zeromq native_packages := native_ccache native_comparisontool qt_native_packages = native_protobuf @@ -12,7 +10,6 @@ qt_i686_linux_packages:=$(qt_x86_64_linux_packages) qt_darwin_packages=qt qt_mingw32_packages=qt - wallet_packages=bdb upnp_packages=miniupnpc diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index f8901f72c2..01146c26f6 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -1,15 +1,22 @@ package=zeromq -$(package)_version=4.1.4 -$(package)_download_path=http://download.zeromq.org +$(package)_version=4.1.5 +$(package)_download_path=https://github.com/zeromq/zeromq4-1/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=e99f44fde25c2e4cb84ce440f87ca7d3fe3271c2b8cfbc67d55e4de25e6fe378 +$(package)_sha256_hash=04aac57f081ffa3a2ee5ed04887be9e205df3a7ddade0027460b8042432bdbcf +$(package)_patches=9114d3957725acd34aa8b8d011585812f3369411.patch 9e6745c12e0b100cd38acecc16ce7db02905e27c.patch define $(package)_set_vars - $(package)_config_opts=--without-documentation --disable-shared --without-libsodium + $(package)_config_opts=--without-documentation --disable-shared --without-libsodium --disable-curve $(package)_config_opts_linux=--with-pic $(package)_cxxflags=-std=c++11 endef +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/9114d3957725acd34aa8b8d011585812f3369411.patch && \ + patch -p1 < $($(package)_patch_dir)/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch && \ + ./autogen.sh +endef + define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/depends/patches/native_biplist/sorted_list.patch b/depends/patches/native_biplist/sorted_list.patch new file mode 100644 index 0000000000..89abdb1b71 --- /dev/null +++ b/depends/patches/native_biplist/sorted_list.patch @@ -0,0 +1,29 @@ +--- a/biplist/__init__.py 2014-10-26 19:03:11.000000000 +0000 ++++ b/biplist/__init__.py 2016-07-19 19:30:17.663521999 +0000 +@@ -541,7 +541,7 @@ + return HashableWrapper(n) + elif isinstance(root, dict): + n = {} +- for key, value in iteritems(root): ++ for key, value in sorted(iteritems(root)): + n[self.wrapRoot(key)] = self.wrapRoot(value) + return HashableWrapper(n) + elif isinstance(root, list): +@@ -616,7 +616,7 @@ + elif isinstance(obj, dict): + size = proc_size(len(obj)) + self.incrementByteCount('dictBytes', incr=1+size) +- for key, value in iteritems(obj): ++ for key, value in sorted(iteritems(obj)): + check_key(key) + self.computeOffsets(key, asReference=True) + self.computeOffsets(value, asReference=True) +@@ -714,7 +714,7 @@ + keys = [] + values = [] + objectsToWrite = [] +- for key, value in iteritems(obj): ++ for key, value in sorted(iteritems(obj)): + keys.append(key) + values.append(value) + for key in keys: diff --git a/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch b/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch new file mode 100644 index 0000000000..f704b3d94f --- /dev/null +++ b/depends/patches/zeromq/9114d3957725acd34aa8b8d011585812f3369411.patch @@ -0,0 +1,22 @@ +From 9114d3957725acd34aa8b8d011585812f3369411 Mon Sep 17 00:00:00 2001 +From: Jeroen Ooms <jeroenooms@gmail.com> +Date: Tue, 20 Oct 2015 13:10:38 +0200 +Subject: [PATCH] enable static libraries on mingw + +--- + configure.ac | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/configure.ac b/configure.ac +index 393505b..e92131a 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -265,7 +265,7 @@ case "${host_os}" in + libzmq_dso_visibility="no" + + if test "x$enable_static" = "xyes"; then +- AC_MSG_ERROR([Building static libraries is not supported under MinGW32]) ++ CPPFLAGS="-DZMQ_STATIC" + fi + + # Set FD_SETSIZE to 1024
\ No newline at end of file diff --git a/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch b/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch new file mode 100644 index 0000000000..9aff2c179a --- /dev/null +++ b/depends/patches/zeromq/9e6745c12e0b100cd38acecc16ce7db02905e27c.patch @@ -0,0 +1,22 @@ +From 9e6745c12e0b100cd38acecc16ce7db02905e27c Mon Sep 17 00:00:00 2001 +From: David Millard <dmillard10@gmail.com> +Date: Tue, 10 May 2016 13:53:53 -0700 +Subject: [PATCH] Fix autotools for static MinGW builds + +--- + configure.ac | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/configure.ac b/configure.ac +index 5a0fa14..def6ea7 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -259,7 +259,7 @@ case "${host_os}" in + libzmq_dso_visibility="no" + + if test "x$enable_static" = "xyes"; then +- CPPFLAGS="-DZMQ_STATIC" ++ CPPFLAGS="-DZMQ_STATIC $CPPFLAGS" + fi + + # Set FD_SETSIZE to 1024
\ No newline at end of file diff --git a/doc/Doxyfile b/doc/Doxyfile index 428fba98e1..22850db835 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -34,7 +34,7 @@ PROJECT_NAME = Bitcoin # This could be handy for archiving the generated documentation or # if some version control system is used. -PROJECT_NUMBER = 0.12.99 +PROJECT_NUMBER = 0.13.99 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer diff --git a/doc/README.md b/doc/README.md index c30f29452b..09a507c9ce 100644 --- a/doc/README.md +++ b/doc/README.md @@ -1,4 +1,4 @@ -Bitcoin Core 0.12.99 +Bitcoin Core 0.13.99 ===================== Setup diff --git a/doc/README_osx.md b/doc/README_osx.md index aed3cd97e1..6a5c672277 100644 --- a/doc/README_osx.md +++ b/doc/README_osx.md @@ -22,7 +22,7 @@ These tools inject timestamps by default, which produce non-deterministic binaries. The ZERO_AR_DATE environment variable is used to disable that. This version of cctools has been patched to use the current version of clang's -headers and and its libLTO.so rather than those from llvmgcc, as it was +headers and its libLTO.so rather than those from llvmgcc, as it was originally done in toolchain4. To complicate things further, all builds must target an Apple SDK. These SDKs diff --git a/doc/README_windows.txt b/doc/README_windows.txt index 2d1c4503c9..74a05138a4 100644 --- a/doc/README_windows.txt +++ b/doc/README_windows.txt @@ -1,4 +1,4 @@ -Bitcoin Core 0.12.99
+Bitcoin Core 0.13.99
=====================
Intro
diff --git a/doc/gitian-building.md b/doc/gitian-building.md index 7796a5fc9c..938f92ff12 100644 --- a/doc/gitian-building.md +++ b/doc/gitian-building.md @@ -55,7 +55,7 @@ In the VirtualBox GUI click "New" and choose the following parameters in the wiz ![](gitian-building/create_vm_memsize.png) -- Memory Size: at least 1024MB, anything less will really slow down the build. +- Memory Size: at least 3000MB, anything less and the build might not complete. ![](gitian-building/create_vm_hard_disk.png) diff --git a/doc/gitian-building/create_vm_memsize.png b/doc/gitian-building/create_vm_memsize.png Binary files differindex 5abfee5337..6f42cda73f 100644 --- a/doc/gitian-building/create_vm_memsize.png +++ b/doc/gitian-building/create_vm_memsize.png diff --git a/doc/release-notes.md b/doc/release-notes.md index 3d2baaaaea..b99192ae97 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -31,7 +31,7 @@ libraries such as Qt are no longer being tested on XP. We do not have time nor resources to provide support for an OS that is end-of-life. From 0.13.0 on, Windows XP is no longer supported. Users are -suggested to upgrade to a newer verion of Windows, or install an alternative OS +suggested to upgrade to a newer version of Windows, or install an alternative OS that is supported. No attempt is made to prevent installing or running the software on Windows XP, @@ -41,81 +41,7 @@ report issues about Windows XP to the issue tracker. Notable changes =============== -Database cache memory increased --------------------------------- - -As a result of growth of the UTXO set, performance with the prior default -database cache of 100 MiB has suffered. -For this reason the default was changed to 300 MiB in this release. - -For nodes on low-memory systems, the database cache can be changed back to -100 MiB (or to another value) by either: - -- Adding `dbcache=100` in bitcoin.conf -- Changing it in the GUI under `Options → Size of database cache` - -Note that the database cache setting has the most performance impact -during initial sync of a node, and when catching up after downtime. - -bitcoin-cli: arguments privacy --------------------------------- - -The RPC command line client gained a new argument, `-stdin` -to read extra arguments from standard input, one per line until EOF/Ctrl-D. -For example: - - $ echo -e "mysecretcode\n120" | src/bitcoin-cli -stdin walletpassphrase - -It is recommended to use this for sensitive information such as wallet -passphrases, as command-line arguments can usually be read from the process -table by any user on the system. - -RPC low-level changes ----------------------- - -- `gettxoutsetinfo` UTXO hash (`hash_serialized`) has changed. There was a divergence between - 32-bit and 64-bit platforms, and the txids were missing in the hashed data. This has been - fixed, but this means that the output will be different than from previous versions. - -- Full UTF-8 support in the RPC API. Non-ASCII characters in, for example, - wallet labels have always been malformed because they weren't taken into account - properly in JSON RPC processing. This is no longer the case. This also affects - the GUI debug console. - -C++11 and Python 3 -------------------- - -Various code modernizations have been done. The Bitcoin Core code base has -started using C++11. This means that a C++11-capable compiler is now needed for -building. Effectively this means GCC 4.7 or higher, or Clang 3.3 or higher. - -When cross-compiling for a target that doesn't have C++11 libraries, configure with -`./configure --enable-glibc-back-compat ... LDFLAGS=-static-libstdc++`. - -For running the functional tests in `qa/rpc-tests`, Python3.4 or higher is now -required. - -Linux ARM builds ------------------- - -Due to popular request, Linux ARM builds have been added to the uploaded -executables. - -The following extra files can be found in the download directory or torrent: - -- `bitcoin-${VERSION}-arm-linux-gnueabihf.tar.gz`: Linux binaries for the most - common 32-bit ARM architecture. -- `bitcoin-${VERSION}-aarch64-linux-gnu.tar.gz`: Linux binaries for the most - common 64-bit ARM architecture. - -ARM builds are still experimental. If you have problems on a certain device or -Linux distribution combination please report them on the bug tracker, it may be -possible to resolve them. - -Note that Android is not considered ARM Linux in this context. The executables -are not expected to work out of the box on Android. - -0.13.0 Change log +0.14.0 Change log ================= Detailed release notes follow. This overview includes changes that affect @@ -125,83 +51,28 @@ git merge commit are mentioned. ### RPC and REST -Asm script outputs replacements for OP_NOP2 and OP_NOP3 -------------------------------------------------------- - -OP_NOP2 has been renamed to OP_CHECKLOCKTIMEVERIFY by [BIP -65](https://github.com/bitcoin/bips/blob/master/bip-0065.mediawiki) - -OP_NOP3 has been renamed to OP_CHECKSEQUENCEVERIFY by [BIP -112](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki) - -The following outputs are affected by this change: -- RPC `getrawtransaction` (in verbose mode) -- RPC `decoderawtransaction` -- RPC `decodescript` -- REST `/rest/tx/` (JSON format) -- REST `/rest/block/` (JSON format when including extended tx details) -- `bitcoin-tx -json` - -New mempool information RPC calls ---------------------------------- - -RPC calls have been added to output detailed statistics for individual mempool -entries, as well as to calculate the in-mempool ancestors or descendants of a -transaction: see `getmempoolentry`, `getmempoolancestors`, `getmempooldescendants`. - -### ZMQ - -Each ZMQ notification now contains an up-counting sequence number that allows -listeners to detect lost notifications. -The sequence number is always the last element in a multi-part ZMQ notification and -therefore backward compatible. -Each message type has its own counter. -(https://github.com/bitcoin/bitcoin/pull/7762) - ### Configuration and command-line options ### Block and transaction handling ### P2P protocol and network code -The p2p alert system has been removed in #7692 and the 'alert' message is no longer supported. - - -Fee filtering of invs (BIP 133) ------------------------------------- - -The optional new p2p message "feefilter" is implemented and the protocol -version is bumped to 70013. Upon receiving a feefilter message from a peer, -a node will not send invs for any transactions which do not meet the filter -feerate. [BIP 133](https://github.com/bitcoin/bips/blob/master/bip-0133.mediawiki) - ### Validation ### Build system ### Wallet -Hierarchical Deterministic Key Generation ------------------------------------------ -Newly created wallets will use hierarchical deterministic key generation -according to BIP32 (keypath m/0'/0'/k'). -Existing wallets will still use traditional key generation. - -Backups of HD wallets, regardless of when they have been created, can -therefore be used to re-generate all possible private keys, even the -ones which haven't already been generated during the time of the backup. - -HD key generation for new wallets can be disabled by `-usehd=0`. Keep in -mind that this flag only has affect on newly created wallets. -You can't disable HD key generation once you have created a HD wallet. - -There is no distinction between internal (change) and external keys. - -[Pull request](https://github.com/bitcoin/bitcoin/pull/8035/files), [BIP 32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) - ### GUI ### Tests ### Miscellaneous +Credits +======= + +Thanks to everyone who directly contributed to this release: + + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 35ee1edae1..41c1ac8556 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -112,16 +112,16 @@ The gbuild invocations below <b>DO NOT DO THIS</b> by default. ### Build and sign Bitcoin Core for Linux, Windows, and OS X: pushd ./gitian-builder - ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml + ./bin/gbuild --memory 3000 --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-linux --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../ - ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win.yml + ./bin/gbuild --memory 3000 --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-win-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win.yml mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ../ - ./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml + ./bin/gbuild --memory 3000 --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml ./bin/gsign --signer $SIGNER --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../ diff --git a/doc/translation_process.md b/doc/translation_process.md index 310d560b36..d8a85292e8 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -94,7 +94,7 @@ When new plurals are added to the source file, it's important to do the followin 7. Save the source file ### Translating a new language -To create a new language template, you will need to edit the languages manifest file `src/qt/bitcoin.qrc` and add a new entry. Below is an example of the english language entry. +To create a new language template, you will need to edit the languages manifest file `src/qt/bitcoin.qrc` and add a new entry. Below is an example of the English language entry. ```xml <qresource prefix="/translations"> diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 11b83bac14..f65a3eefc3 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -107,6 +107,7 @@ testScripts = [ 'bip68-112-113-p2p.py', 'wallet.py', 'wallet-hd.py', + 'wallet-dump.py', 'listtransactions.py', 'receivedby.py', 'mempool_resurrect_test.py', @@ -141,6 +142,7 @@ testScripts = [ 'segwit.py', 'importprunedfunds.py', 'signmessages.py', + 'p2p-compactblocks.py', ] if ENABLE_ZMQ: testScripts.append('zmq_test.py') @@ -158,7 +160,7 @@ testScriptsExt = [ 'txn_clone.py --mineblock', 'forknotify.py', 'invalidateblock.py', -# 'rpcbind_test.py', #temporary, bug in libevent, see #6655 + 'rpcbind_test.py', 'smartfees.py', 'maxblocksinflight.py', 'p2p-acceptblock.py', diff --git a/qa/rpc-tests/bip65-cltv-p2p.py b/qa/rpc-tests/bip65-cltv-p2p.py index 754b6873b7..e903b2fbf0 100755 --- a/qa/rpc-tests/bip65-cltv-p2p.py +++ b/qa/rpc-tests/bip65-cltv-p2p.py @@ -71,9 +71,9 @@ class BIP65Test(ComparisonTestFramework): self.nodeaddress = self.nodes[0].getnewaddress() self.last_block_time = int(time.time()) - ''' 98 more version 3 blocks ''' + ''' 398 more version 3 blocks ''' test_blocks = [] - for i in range(98): + for i in range(398): block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 3 block.rehash() @@ -118,24 +118,6 @@ class BIP65Test(ComparisonTestFramework): height += 1 yield TestInstance([[block, True]]) - ''' - Check that the new CLTV rules are enforced in the 751st version 4 - block. - ''' - spendtx = self.create_transaction(self.nodes[0], - self.coinbase_blocks[1], self.nodeaddress, 1.0) - cltv_invalidate(spendtx) - spendtx.rehash() - - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 4 - block.vtx.append(spendtx) - block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() - block.solve() - self.last_block_time += 1 - yield TestInstance([[block, False]]) - ''' Mine 199 new version blocks on last valid tip ''' test_blocks = [] for i in range(199): @@ -169,6 +151,24 @@ class BIP65Test(ComparisonTestFramework): height += 1 yield TestInstance([[block, True]]) + ''' + Check that the new CLTV rules are enforced in the 951st version 4 + block. + ''' + spendtx = self.create_transaction(self.nodes[0], + self.coinbase_blocks[1], self.nodeaddress, 1.0) + cltv_invalidate(spendtx) + spendtx.rehash() + + block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) + block.nVersion = 4 + block.vtx.append(spendtx) + block.hashMerkleRoot = block.calc_merkle_root() + block.rehash() + block.solve() + self.last_block_time += 1 + yield TestInstance([[block, False]]) + ''' Mine 1 old version block, should be invalid ''' block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 3 diff --git a/qa/rpc-tests/bip65-cltv.py b/qa/rpc-tests/bip65-cltv.py index abba7fc20e..baa77b92a0 100755 --- a/qa/rpc-tests/bip65-cltv.py +++ b/qa/rpc-tests/bip65-cltv.py @@ -30,7 +30,8 @@ class BIP65Test(BitcoinTestFramework): cnt = self.nodes[0].getblockcount() # Mine some old-version blocks - self.nodes[1].generate(100) + self.nodes[1].generate(200) + cnt += 100 self.sync_all() if (self.nodes[0].getblockcount() != cnt + 100): raise AssertionError("Failed to mine 100 version=3 blocks") diff --git a/qa/rpc-tests/bipdersig-p2p.py b/qa/rpc-tests/bipdersig-p2p.py index 4e4936a4ae..3bad5af5e6 100755 --- a/qa/rpc-tests/bipdersig-p2p.py +++ b/qa/rpc-tests/bipdersig-p2p.py @@ -79,9 +79,9 @@ class BIP66Test(ComparisonTestFramework): self.nodeaddress = self.nodes[0].getnewaddress() self.last_block_time = int(time.time()) - ''' 98 more version 2 blocks ''' + ''' 298 more version 2 blocks ''' test_blocks = [] - for i in range(98): + for i in range(298): block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 2 block.rehash() @@ -124,25 +124,7 @@ class BIP66Test(ComparisonTestFramework): self.last_block_time += 1 self.tip = block.sha256 height += 1 - yield TestInstance([[block, True]]) - - ''' - Check that the new DERSIG rules are enforced in the 751st version 3 - block. - ''' - spendtx = self.create_transaction(self.nodes[0], - self.coinbase_blocks[1], self.nodeaddress, 1.0) - unDERify(spendtx) - spendtx.rehash() - - block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) - block.nVersion = 3 - block.vtx.append(spendtx) - block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() - block.solve() - self.last_block_time += 1 - yield TestInstance([[block, False]]) + yield TestInstance([[block, True]]) ''' Mine 199 new version blocks on last valid tip ''' test_blocks = [] @@ -177,6 +159,24 @@ class BIP66Test(ComparisonTestFramework): height += 1 yield TestInstance([[block, True]]) + ''' + Check that the new DERSIG rules are enforced in the 951st version 3 + block. + ''' + spendtx = self.create_transaction(self.nodes[0], + self.coinbase_blocks[1], self.nodeaddress, 1.0) + unDERify(spendtx) + spendtx.rehash() + + block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) + block.nVersion = 3 + block.vtx.append(spendtx) + block.hashMerkleRoot = block.calc_merkle_root() + block.rehash() + block.solve() + self.last_block_time += 1 + yield TestInstance([[block, False]]) + ''' Mine 1 old version block, should be invalid ''' block = create_block(self.tip, create_coinbase(height), self.last_block_time + 1) block.nVersion = 2 diff --git a/qa/rpc-tests/keypool.py b/qa/rpc-tests/keypool.py index c75303ecbf..fa39476568 100755 --- a/qa/rpc-tests/keypool.py +++ b/qa/rpc-tests/keypool.py @@ -12,6 +12,11 @@ class KeyPoolTest(BitcoinTestFramework): def run_test(self): nodes = self.nodes + addr_before_encrypting = nodes[0].getnewaddress() + addr_before_encrypting_data = nodes[0].validateaddress(addr_before_encrypting) + wallet_info_old = nodes[0].getwalletinfo() + assert(addr_before_encrypting_data['hdmasterkeyid'] == wallet_info_old['hdmasterkeyid']) + # Encrypt wallet and wait to terminate nodes[0].encryptwallet('test') bitcoind_processes[0].wait() @@ -19,6 +24,11 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0] = start_node(0, self.options.tmpdir) # Keep creating keys addr = nodes[0].getnewaddress() + addr_data = nodes[0].validateaddress(addr) + wallet_info = nodes[0].getwalletinfo() + assert(addr_before_encrypting_data['hdmasterkeyid'] != wallet_info['hdmasterkeyid']) + assert(addr_data['hdmasterkeyid'] == wallet_info['hdmasterkeyid']) + try: addr = nodes[0].getnewaddress() raise AssertionError('Keypool should be exhausted after one address') diff --git a/qa/rpc-tests/p2p-compactblocks.py b/qa/rpc-tests/p2p-compactblocks.py new file mode 100755 index 0000000000..7fe7ecc16c --- /dev/null +++ b/qa/rpc-tests/p2p-compactblocks.py @@ -0,0 +1,608 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.mininode import * +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.blocktools import create_block, create_coinbase +from test_framework.siphash import siphash256 +from test_framework.script import CScript, OP_TRUE + +''' +CompactBlocksTest -- test compact blocks (BIP 152) +''' + + +# TestNode: A peer we use to send messages to bitcoind, and store responses. +class TestNode(SingleNodeConnCB): + def __init__(self): + SingleNodeConnCB.__init__(self) + self.last_sendcmpct = None + self.last_headers = None + self.last_inv = None + self.last_cmpctblock = None + self.block_announced = False + self.last_getdata = None + self.last_getblocktxn = None + self.last_block = None + self.last_blocktxn = None + + def on_sendcmpct(self, conn, message): + self.last_sendcmpct = message + + def on_block(self, conn, message): + self.last_block = message + + def on_cmpctblock(self, conn, message): + self.last_cmpctblock = message + self.block_announced = True + + def on_headers(self, conn, message): + self.last_headers = message + self.block_announced = True + + def on_inv(self, conn, message): + self.last_inv = message + self.block_announced = True + + def on_getdata(self, conn, message): + self.last_getdata = message + + def on_getblocktxn(self, conn, message): + self.last_getblocktxn = message + + def on_blocktxn(self, conn, message): + self.last_blocktxn = message + + # Requires caller to hold mininode_lock + def received_block_announcement(self): + return self.block_announced + + def clear_block_announcement(self): + with mininode_lock: + self.block_announced = False + self.last_inv = None + self.last_headers = None + self.last_cmpctblock = None + + def get_headers(self, locator, hashstop): + msg = msg_getheaders() + msg.locator.vHave = locator + msg.hashstop = hashstop + self.connection.send_message(msg) + + def send_header_for_blocks(self, new_blocks): + headers_message = msg_headers() + headers_message.headers = [CBlockHeader(b) for b in new_blocks] + self.send_message(headers_message) + + +class CompactBlocksTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 + self.utxos = [] + + def setup_network(self): + self.nodes = [] + + # Turn off segwit in this test, as compact blocks don't currently work + # with segwit. (After BIP 152 is updated to support segwit, we can + # test behavior with and without segwit enabled by adding a second node + # to the test.) + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [["-debug", "-logtimemicros=1", "-bip9params=segwit:0:0"]]) + + def build_block_on_tip(self): + height = self.nodes[0].getblockcount() + tip = self.nodes[0].getbestblockhash() + mtp = self.nodes[0].getblockheader(tip)['mediantime'] + block = create_block(int(tip, 16), create_coinbase(height + 1), mtp + 1) + block.solve() + return block + + # Create 10 more anyone-can-spend utxo's for testing. + def make_utxos(self): + block = self.build_block_on_tip() + self.test_node.send_and_ping(msg_block(block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) + self.nodes[0].generate(100) + + total_value = block.vtx[0].vout[0].nValue + out_value = total_value // 10 + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(block.vtx[0].sha256, 0), b'')) + for i in range(10): + tx.vout.append(CTxOut(out_value, CScript([OP_TRUE]))) + tx.rehash() + + block2 = self.build_block_on_tip() + block2.vtx.append(tx) + block2.hashMerkleRoot = block2.calc_merkle_root() + block2.solve() + self.test_node.send_and_ping(msg_block(block2)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block2.sha256) + self.utxos.extend([[tx.sha256, i, out_value] for i in range(10)]) + return + + # Test "sendcmpct": + # - No compact block announcements or getdata(MSG_CMPCT_BLOCK) unless + # sendcmpct is sent. + # - If sendcmpct is sent with version > 0, the message is ignored. + # - If sendcmpct is sent with boolean 0, then block announcements are not + # made with compact blocks. + # - If sendcmpct is then sent with boolean 1, then new block announcements + # are made with compact blocks. + def test_sendcmpct(self): + print("Testing SENDCMPCT p2p message... ") + + # Make sure we get a version 0 SENDCMPCT message from our peer + def received_sendcmpct(): + return (self.test_node.last_sendcmpct is not None) + got_message = wait_until(received_sendcmpct, timeout=30) + assert(got_message) + assert_equal(self.test_node.last_sendcmpct.version, 1) + + tip = int(self.nodes[0].getbestblockhash(), 16) + + def check_announcement_of_new_block(node, peer, predicate): + self.test_node.clear_block_announcement() + node.generate(1) + got_message = wait_until(peer.received_block_announcement, timeout=30) + assert(got_message) + with mininode_lock: + assert(predicate) + + # We shouldn't get any block announcements via cmpctblock yet. + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Try one more time, this time after requesting headers. + self.test_node.clear_block_announcement() + self.test_node.get_headers(locator=[tip], hashstop=0) + wait_until(self.test_node.received_block_announcement, timeout=30) + self.test_node.clear_block_announcement() + + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_inv is not None) + + # Now try a SENDCMPCT message with too-high version + sendcmpct = msg_sendcmpct() + sendcmpct.version = 2 + self.test_node.send_message(sendcmpct) + + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Now try a SENDCMPCT message with valid version, but announce=False + self.test_node.send_message(msg_sendcmpct()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None) + + # Finally, try a SENDCMPCT message with announce=True + sendcmpct.version = 1 + sendcmpct.announce = True + self.test_node.send_message(sendcmpct) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Try one more time + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Try one more time, after turning on sendheaders + self.test_node.send_message(msg_sendheaders()) + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is not None) + + # Now turn off announcements + sendcmpct.announce = False + check_announcement_of_new_block(self.nodes[0], self.test_node, lambda: self.test_node.last_cmpctblock is None and self.test_node.last_headers is not None) + + # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. + def test_invalid_cmpctblock_message(self): + print("Testing invalid index in cmpctblock message...") + self.nodes[0].generate(101) + block = self.build_block_on_tip() + + cmpct_block = P2PHeaderAndShortIDs() + cmpct_block.header = CBlockHeader(block) + cmpct_block.prefilled_txn_length = 1 + # This index will be too high + prefilled_txn = PrefilledTransaction(1, block.vtx[0]) + cmpct_block.prefilled_txn = [prefilled_txn] + self.test_node.send_and_ping(msg_cmpctblock(cmpct_block)) + assert(int(self.nodes[0].getbestblockhash(), 16) == block.hashPrevBlock) + + # Compare the generated shortids to what we expect based on BIP 152, given + # bitcoind's choice of nonce. + def test_compactblock_construction(self): + print("Testing compactblock headers and shortIDs are correct...") + + # Generate a bunch of transactions. + self.nodes[0].generate(101) + num_transactions = 25 + address = self.nodes[0].getnewaddress() + for i in range(num_transactions): + self.nodes[0].sendtoaddress(address, 0.1) + + # Now mine a block, and look at the resulting compact block. + self.test_node.clear_block_announcement() + block_hash = int(self.nodes[0].generate(1)[0], 16) + + # Store the raw block in our internal format. + block = FromHex(CBlock(), self.nodes[0].getblock("%02x" % block_hash, False)) + [tx.calc_sha256() for tx in block.vtx] + block.rehash() + + # Don't care which type of announcement came back for this test; just + # request the compact block if we didn't get one yet. + wait_until(self.test_node.received_block_announcement, timeout=30) + + with mininode_lock: + if self.test_node.last_cmpctblock is None: + self.test_node.clear_block_announcement() + inv = CInv(4, block_hash) # 4 == "CompactBlock" + self.test_node.send_message(msg_getdata([inv])) + + wait_until(self.test_node.received_block_announcement, timeout=30) + + # Now we should have the compactblock + header_and_shortids = None + with mininode_lock: + assert(self.test_node.last_cmpctblock is not None) + # Convert the on-the-wire representation to absolute indexes + header_and_shortids = HeaderAndShortIDs(self.test_node.last_cmpctblock.header_and_shortids) + + # Check that we got the right block! + header_and_shortids.header.calc_sha256() + assert_equal(header_and_shortids.header.sha256, block_hash) + + # Make sure the prefilled_txn appears to have included the coinbase + assert(len(header_and_shortids.prefilled_txn) >= 1) + assert_equal(header_and_shortids.prefilled_txn[0].index, 0) + + # Check that all prefilled_txn entries match what's in the block. + for entry in header_and_shortids.prefilled_txn: + entry.tx.calc_sha256() + assert_equal(entry.tx.sha256, block.vtx[entry.index].sha256) + + # Check that the cmpctblock message announced all the transactions. + assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) + + # And now check that all the shortids are as expected as well. + # Determine the siphash keys to use. + [k0, k1] = header_and_shortids.get_siphash_keys() + + index = 0 + while index < len(block.vtx): + if (len(header_and_shortids.prefilled_txn) > 0 and + header_and_shortids.prefilled_txn[0].index == index): + # Already checked prefilled transactions above + header_and_shortids.prefilled_txn.pop(0) + else: + shortid = calculate_shortid(k0, k1, block.vtx[index].sha256) + assert_equal(shortid, header_and_shortids.shortids[0]) + header_and_shortids.shortids.pop(0) + index += 1 + + # Test that bitcoind requests compact blocks when we announce new blocks + # via header or inv, and that responding to getblocktxn causes the block + # to be successfully reconstructed. + def test_compactblock_requests(self): + print("Testing compactblock requests... ") + + # Try announcing a block with an inv or header, expect a compactblock + # request + for announce in ["inv", "header"]: + block = self.build_block_on_tip() + with mininode_lock: + self.test_node.last_getdata = None + + if announce == "inv": + self.test_node.send_message(msg_inv([CInv(2, block.sha256)])) + else: + self.test_node.send_header_for_blocks([block]) + success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=30) + assert(success) + assert_equal(len(self.test_node.last_getdata.inv), 1) + assert_equal(self.test_node.last_getdata.inv[0].type, 4) + assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256) + + # Send back a compactblock message that omits the coinbase + comp_block = HeaderAndShortIDs() + comp_block.header = CBlockHeader(block) + comp_block.nonce = 0 + comp_block.shortids = [1] # this is useless, and wrong + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + # Expect a getblocktxn message. + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [0]) # should be a coinbase request + + # Send the coinbase, and verify that the tip advances. + msg = msg_blocktxn() + msg.block_transactions.blockhash = block.sha256 + msg.block_transactions.transactions = [block.vtx[0]] + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Create a chain of transactions from given utxo, and add to a new block. + def build_block_with_transactions(self, utxo, num_transactions): + block = self.build_block_on_tip() + + for i in range(num_transactions): + tx = CTransaction() + tx.vin.append(CTxIn(COutPoint(utxo[0], utxo[1]), b'')) + tx.vout.append(CTxOut(utxo[2] - 1000, CScript([OP_TRUE]))) + tx.rehash() + utxo = [tx.sha256, 0, tx.vout[0].nValue] + block.vtx.append(tx) + + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + return block + + # Test that we only receive getblocktxn requests for transactions that the + # node needs, and that responding to them causes the block to be + # reconstructed. + def test_getblocktxn_requests(self): + print("Testing getblocktxn requests...") + + # First try announcing compactblocks that won't reconstruct, and verify + # that we receive getblocktxn messages back. + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [1, 2, 3, 4, 5]) + msg = msg_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, block.vtx[1:]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + + # Now try interspersing the prefilled transactions + comp_block.initialize_from_block(block, prefill_list=[0, 1, 5]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [2, 3, 4]) + msg.block_transactions = BlockTransactions(block.sha256, block.vtx[2:5]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Now try giving one transaction ahead of time. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 5) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + self.test_node.send_and_ping(msg_tx(block.vtx[1])) + assert(block.vtx[1].hash in self.nodes[0].getrawmempool()) + + # Prefill 4 out of the 6 transactions, and verify that only the one + # that was not in the mempool is requested. + comp_block.initialize_from_block(block, prefill_list=[0, 2, 3, 4]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [5]) + + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]]) + self.test_node.send_and_ping(msg) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Now provide all transactions to the node before the block is + # announced and verify reconstruction happens immediately. + utxo = self.utxos.pop(0) + block = self.build_block_with_transactions(utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + for tx in block.vtx[1:]: + self.test_node.send_message(msg_tx(tx)) + self.test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = self.nodes[0].getrawmempool() + for tx in block.vtx[1:]: + assert(tx.hash in mempool) + + # Clear out last request. + with mininode_lock: + self.test_node.last_getblocktxn = None + + # Send compact block + comp_block.initialize_from_block(block, prefill_list=[0]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + with mininode_lock: + # Shouldn't have gotten a request for any transaction + assert(self.test_node.last_getblocktxn is None) + # Tip should have updated + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + # Incorrectly responding to a getblocktxn shouldn't cause the block to be + # permanently failed. + def test_incorrect_blocktxn_response(self): + print("Testing handling of incorrect blocktxn responses...") + + if (len(self.utxos) == 0): + self.make_utxos() + utxo = self.utxos.pop(0) + + block = self.build_block_with_transactions(utxo, 10) + self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) + # Relay the first 5 transactions from the block in advance + for tx in block.vtx[1:6]: + self.test_node.send_message(msg_tx(tx)) + self.test_node.sync_with_ping() + # Make sure all transactions were accepted. + mempool = self.nodes[0].getrawmempool() + for tx in block.vtx[1:6]: + assert(tx.hash in mempool) + + # Send compact block + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block, prefill_list=[0]) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + absolute_indexes = [] + with mininode_lock: + assert(self.test_node.last_getblocktxn is not None) + absolute_indexes = self.test_node.last_getblocktxn.block_txn_request.to_absolute() + assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) + + # Now give an incorrect response. + # Note that it's possible for bitcoind to be smart enough to know we're + # lying, since it could check to see if the shortid matches what we're + # sending, and eg disconnect us for misbehavior. If that behavior + # change were made, we could just modify this test by having a + # different peer provide the block further down, so that we're still + # verifying that the block isn't marked bad permanently. This is good + # enough for now. + msg = msg_blocktxn() + msg.block_transactions = BlockTransactions(block.sha256, [block.vtx[5]] + block.vtx[7:]) + self.test_node.send_and_ping(msg) + + # Tip should not have updated + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.hashPrevBlock) + + # We should receive a getdata request + success = wait_until(lambda: self.test_node.last_getdata is not None, timeout=10) + assert(success) + assert_equal(len(self.test_node.last_getdata.inv), 1) + assert_equal(self.test_node.last_getdata.inv[0].type, 2) + assert_equal(self.test_node.last_getdata.inv[0].hash, block.sha256) + + # Deliver the block + self.test_node.send_and_ping(msg_block(block)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256) + + def test_getblocktxn_handler(self): + print("Testing getblocktxn handler...") + + # bitcoind won't respond for blocks whose height is more than 15 blocks + # deep. + MAX_GETBLOCKTXN_DEPTH = 15 + chain_height = self.nodes[0].getblockcount() + current_height = chain_height + while (current_height >= chain_height - MAX_GETBLOCKTXN_DEPTH): + block_hash = self.nodes[0].getblockhash(current_height) + block = FromHex(CBlock(), self.nodes[0].getblock(block_hash, False)) + + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), []) + num_to_request = random.randint(1, len(block.vtx)) + msg.block_txn_request.from_absolute(sorted(random.sample(range(len(block.vtx)), num_to_request))) + self.test_node.send_message(msg) + success = wait_until(lambda: self.test_node.last_blocktxn is not None, timeout=10) + assert(success) + + [tx.calc_sha256() for tx in block.vtx] + with mininode_lock: + assert_equal(self.test_node.last_blocktxn.block_transactions.blockhash, int(block_hash, 16)) + all_indices = msg.block_txn_request.to_absolute() + for index in all_indices: + tx = self.test_node.last_blocktxn.block_transactions.transactions.pop(0) + tx.calc_sha256() + assert_equal(tx.sha256, block.vtx[index].sha256) + self.test_node.last_blocktxn = None + current_height -= 1 + + # Next request should be ignored, as we're past the allowed depth. + block_hash = self.nodes[0].getblockhash(current_height) + msg.block_txn_request = BlockTransactionsRequest(int(block_hash, 16), [0]) + self.test_node.send_and_ping(msg) + with mininode_lock: + assert_equal(self.test_node.last_blocktxn, None) + + def test_compactblocks_not_at_tip(self): + print("Testing compactblock requests/announcements not at chain tip...") + + # Test that requesting old compactblocks doesn't work. + MAX_CMPCTBLOCK_DEPTH = 11 + new_blocks = [] + for i in range(MAX_CMPCTBLOCK_DEPTH): + self.test_node.clear_block_announcement() + new_blocks.append(self.nodes[0].generate(1)[0]) + wait_until(self.test_node.received_block_announcement, timeout=30) + + self.test_node.clear_block_announcement() + self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + success = wait_until(lambda: self.test_node.last_cmpctblock is not None, timeout=30) + assert(success) + + self.test_node.clear_block_announcement() + self.nodes[0].generate(1) + wait_until(self.test_node.received_block_announcement, timeout=30) + self.test_node.clear_block_announcement() + self.test_node.send_message(msg_getdata([CInv(4, int(new_blocks[0], 16))])) + success = wait_until(lambda: self.test_node.last_block is not None, timeout=30) + assert(success) + with mininode_lock: + self.test_node.last_block.block.calc_sha256() + assert_equal(self.test_node.last_block.block.sha256, int(new_blocks[0], 16)) + + # Generate an old compactblock, and verify that it's not accepted. + cur_height = self.nodes[0].getblockcount() + hashPrevBlock = int(self.nodes[0].getblockhash(cur_height-5), 16) + block = self.build_block_on_tip() + block.hashPrevBlock = hashPrevBlock + block.solve() + + comp_block = HeaderAndShortIDs() + comp_block.initialize_from_block(block) + self.test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) + + tips = self.nodes[0].getchaintips() + found = False + for x in tips: + if x["hash"] == block.hash: + assert_equal(x["status"], "headers-only") + found = True + break + assert(found) + + # Requesting this block via getblocktxn should silently fail + # (to avoid fingerprinting attacks). + msg = msg_getblocktxn() + msg.block_txn_request = BlockTransactionsRequest(block.sha256, [0]) + with mininode_lock: + self.test_node.last_blocktxn = None + self.test_node.send_and_ping(msg) + with mininode_lock: + assert(self.test_node.last_blocktxn is None) + + def run_test(self): + # Setup the p2p connections and start up the network thread. + self.test_node = TestNode() + + connections = [] + connections.append(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], self.test_node)) + self.test_node.add_connection(connections[0]) + + NetworkThread().start() # Start up network handling in another thread + + # Test logic begins here + self.test_node.wait_for_verack() + + # We will need UTXOs to construct transactions in later tests. + self.make_utxos() + + self.test_sendcmpct() + self.test_compactblock_construction() + self.test_compactblock_requests() + self.test_getblocktxn_requests() + self.test_getblocktxn_handler() + self.test_compactblocks_not_at_tip() + self.test_incorrect_blocktxn_response() + self.test_invalid_cmpctblock_message() + + +if __name__ == '__main__': + CompactBlocksTest().main() diff --git a/qa/rpc-tests/p2p-feefilter.py b/qa/rpc-tests/p2p-feefilter.py index cd0501a314..96d99d38a7 100755 --- a/qa/rpc-tests/p2p-feefilter.py +++ b/qa/rpc-tests/p2p-feefilter.py @@ -62,6 +62,7 @@ class FeeFilterTest(BitcoinTestFramework): def run_test(self): node1 = self.nodes[1] + node0 = self.nodes[0] # Get out of IBD node1.generate(1) sync_blocks(self.nodes) @@ -91,8 +92,17 @@ class FeeFilterTest(BitcoinTestFramework): node1.settxfee(Decimal("0.00010000")) [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] sync_mempools(self.nodes) # must be sure node 0 has received all txs - time.sleep(10) # wait 10 secs to be sure its doesn't relay any - assert(allInvsMatch([], test_node)) + + # Send one transaction from node0 that should be received, so that we + # we can sync the test on receipt (if node1's txs were relayed, they'd + # be received by the time this node0 tx is received). This is + # unfortunately reliant on the current relay behavior where we batch up + # to 35 entries in an inv, which means that when this next transaction + # is eligible for relay, the prior transactions from node1 are eligible + # as well. + node0.settxfee(Decimal("0.00020000")) + txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] + assert(allInvsMatch(txids, test_node)) test_node.clear_invs() # Remove fee filter and check that txs are received again diff --git a/qa/rpc-tests/p2p-segwit.py b/qa/rpc-tests/p2p-segwit.py index b30d41af92..cd02692b1e 100755 --- a/qa/rpc-tests/p2p-segwit.py +++ b/qa/rpc-tests/p2p-segwit.py @@ -1065,12 +1065,12 @@ class SegWitTest(BitcoinTestFramework): assert_equal(wit_block.serialize(False), non_wit_block.serialize()) assert_equal(wit_block.serialize(True), block.serialize(True)) - # Test size, vsize, cost + # Test size, vsize, weight rpc_details = self.nodes[0].getblock(block.hash, True) assert_equal(rpc_details["size"], len(block.serialize(True))) assert_equal(rpc_details["strippedsize"], len(block.serialize(False))) - cost = 3*len(block.serialize(False)) + len(block.serialize(True)) - assert_equal(rpc_details["cost"], cost) + weight = 3*len(block.serialize(False)) + len(block.serialize(True)) + assert_equal(rpc_details["weight"], weight) # Upgraded node should not ask for blocks from unupgraded block4 = self.build_next_block(nVersion=4) @@ -1086,6 +1086,82 @@ class SegWitTest(BitcoinTestFramework): self.old_node.announce_tx_and_wait_for_getdata(block4.vtx[0]) assert(block4.sha256 not in self.old_node.getdataset) + # V0 segwit outputs should be standard after activation, but not before. + def test_standardness_v0(self, segwit_activated): + print("\tTesting standardness of v0 outputs (%s activation)" % ("after" if segwit_activated else "before")) + assert(len(self.utxo)) + + witness_program = CScript([OP_TRUE]) + witness_hash = sha256(witness_program) + scriptPubKey = CScript([OP_0, witness_hash]) + + p2sh_pubkey = hash160(witness_program) + p2sh_scriptPubKey = CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]) + + # First prepare a p2sh output (so that spending it will pass standardness) + p2sh_tx = CTransaction() + p2sh_tx.vin = [CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")] + p2sh_tx.vout = [CTxOut(self.utxo[0].nValue-1000, p2sh_scriptPubKey)] + p2sh_tx.rehash() + + # Mine it on test_node to create the confirmed output. + self.test_node.test_transaction_acceptance(p2sh_tx, with_witness=True, accepted=True) + self.nodes[0].generate(1) + sync_blocks(self.nodes) + + # Now test standardness of v0 P2WSH outputs. + # Start by creating a transaction with two outputs. + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(p2sh_tx.sha256, 0), CScript([witness_program]))] + tx.vout = [CTxOut(p2sh_tx.vout[0].nValue-10000, scriptPubKey)] + tx.vout.append(CTxOut(8000, scriptPubKey)) # Might burn this later + tx.rehash() + + self.std_node.test_transaction_acceptance(tx, with_witness=True, accepted=segwit_activated) + + # Now create something that looks like a P2PKH output. This won't be spendable. + scriptPubKey = CScript([OP_0, hash160(witness_hash)]) + tx2 = CTransaction() + if segwit_activated: + # if tx was accepted, then we spend the second output. + tx2.vin = [CTxIn(COutPoint(tx.sha256, 1), b"")] + tx2.vout = [CTxOut(7000, scriptPubKey)] + tx2.wit.vtxinwit.append(CTxInWitness()) + tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + else: + # if tx wasn't accepted, we just re-spend the p2sh output we started with. + tx2.vin = [CTxIn(COutPoint(p2sh_tx.sha256, 0), CScript([witness_program]))] + tx2.vout = [CTxOut(p2sh_tx.vout[0].nValue-1000, scriptPubKey)] + tx2.rehash() + + self.std_node.test_transaction_acceptance(tx2, with_witness=True, accepted=segwit_activated) + + # Now update self.utxo for later tests. + tx3 = CTransaction() + if segwit_activated: + # tx and tx2 were both accepted. Don't bother trying to reclaim the + # P2PKH output; just send tx's first output back to an anyone-can-spend. + sync_mempools(self.nodes) + tx3.vin = [CTxIn(COutPoint(tx.sha256, 0), b"")] + tx3.vout = [CTxOut(tx.vout[0].nValue-1000, CScript([OP_TRUE]))] + tx3.wit.vtxinwit.append(CTxInWitness()) + tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + tx3.rehash() + self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=True) + else: + # tx and tx2 didn't go anywhere; just clean up the p2sh_tx output. + tx3.vin = [CTxIn(COutPoint(p2sh_tx.sha256, 0), CScript([witness_program]))] + tx3.vout = [CTxOut(p2sh_tx.vout[0].nValue-1000, witness_program)] + tx3.rehash() + self.test_node.test_transaction_acceptance(tx3, with_witness=True, accepted=True) + + self.nodes[0].generate(1) + sync_blocks(self.nodes) + self.utxo.pop(0) + self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) + assert_equal(len(self.nodes[1].getrawmempool()), 0) + + # Verify that future segwit upgraded transactions are non-standard, # but valid in blocks. Can run this before and after segwit activation. def test_segwit_versions(self): @@ -1658,6 +1734,7 @@ class SegWitTest(BitcoinTestFramework): self.test_witness_tx_relay_before_segwit_activation() self.test_block_relay(segwit_activated=False) self.test_p2sh_witness(segwit_activated=False) + self.test_standardness_v0(segwit_activated=False) sync_blocks(self.nodes) @@ -1679,6 +1756,7 @@ class SegWitTest(BitcoinTestFramework): self.test_witness_input_length() self.test_block_relay(segwit_activated=True) self.test_tx_relay_after_segwit_activation() + self.test_standardness_v0(segwit_activated=True) self.test_segwit_versions() self.test_premature_coinbase_witness_spend() self.test_signature_version_1() diff --git a/qa/rpc-tests/rpcbind_test.py b/qa/rpc-tests/rpcbind_test.py index 572273566b..bf1cc87126 100755 --- a/qa/rpc-tests/rpcbind_test.py +++ b/qa/rpc-tests/rpcbind_test.py @@ -5,143 +5,108 @@ # Test for -rpcbind, as well as -rpcallowip and -rpcconnect -# TODO extend this test from the test framework (like all other tests) - import tempfile import traceback +from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.netutil import * -def run_bind_test(tmpdir, allow_ips, connect_to, addresses, expected): - ''' - Start a node with requested rpcallowip and rpcbind parameters, - then try to connect, and check if the set of bound addresses - matches the expected set. - ''' - expected = [(addr_to_hex(addr), port) for (addr, port) in expected] - base_args = ['-disablewallet', '-nolisten'] - if allow_ips: - base_args += ['-rpcallowip=' + x for x in allow_ips] - binds = ['-rpcbind='+addr for addr in addresses] - nodes = start_nodes(self.num_nodes, tmpdir, [base_args + binds], connect_to) - try: - pid = bitcoind_processes[0].pid - assert_equal(set(get_bind_addrs(pid)), set(expected)) - finally: - stop_nodes(nodes) - wait_bitcoinds() - -def run_allowip_test(tmpdir, allow_ips, rpchost, rpcport): - ''' - Start a node with rpcwallow IP, and request getinfo - at a non-localhost IP. - ''' - base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips] - nodes = start_nodes(self.num_nodes, tmpdir, [base_args]) - try: - # connect to node through non-loopback interface - url = "http://rt:rt@%s:%d" % (rpchost, rpcport,) - node = get_rpc_proxy(url, 1) - node.getinfo() - finally: - node = None # make sure connection will be garbage collected and closed - stop_nodes(nodes) - wait_bitcoinds() - - -def run_test(tmpdir): - assert(sys.platform.startswith('linux')) # due to OS-specific network stats queries, this test works only on Linux - # find the first non-loopback interface for testing - non_loopback_ip = None - for name,ip in all_interfaces(): - if ip != '127.0.0.1': - non_loopback_ip = ip - break - if non_loopback_ip is None: - assert(not 'This test requires at least one non-loopback IPv4 interface') - print("Using interface %s for testing" % non_loopback_ip) +class RPCBindTest(BitcoinTestFramework): - defaultport = rpc_port(0) + def __init__(self): + super().__init__() + self.setup_clean_chain = True + self.num_nodes = 1 - # check default without rpcallowip (IPv4 and IPv6 localhost) - run_bind_test(tmpdir, None, '127.0.0.1', [], - [('127.0.0.1', defaultport), ('::1', defaultport)]) - # check default with rpcallowip (IPv6 any) - run_bind_test(tmpdir, ['127.0.0.1'], '127.0.0.1', [], - [('::0', defaultport)]) - # check only IPv4 localhost (explicit) - run_bind_test(tmpdir, ['127.0.0.1'], '127.0.0.1', ['127.0.0.1'], - [('127.0.0.1', defaultport)]) - # check only IPv4 localhost (explicit) with alternative port - run_bind_test(tmpdir, ['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171'], - [('127.0.0.1', 32171)]) - # check only IPv4 localhost (explicit) with multiple alternative ports on same host - run_bind_test(tmpdir, ['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171', '127.0.0.1:32172'], - [('127.0.0.1', 32171), ('127.0.0.1', 32172)]) - # check only IPv6 localhost (explicit) - run_bind_test(tmpdir, ['[::1]'], '[::1]', ['[::1]'], - [('::1', defaultport)]) - # check both IPv4 and IPv6 localhost (explicit) - run_bind_test(tmpdir, ['127.0.0.1'], '127.0.0.1', ['127.0.0.1', '[::1]'], - [('127.0.0.1', defaultport), ('::1', defaultport)]) - # check only non-loopback interface - run_bind_test(tmpdir, [non_loopback_ip], non_loopback_ip, [non_loopback_ip], - [(non_loopback_ip, defaultport)]) - - # Check that with invalid rpcallowip, we are denied - run_allowip_test(tmpdir, [non_loopback_ip], non_loopback_ip, defaultport) - try: - run_allowip_test(tmpdir, ['1.1.1.1'], non_loopback_ip, defaultport) - assert(not 'Connection not denied by rpcallowip as expected') - except ValueError: + def setup_network(self): pass -def main(): - import optparse - - parser = optparse.OptionParser(usage="%prog [options]") - parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true", - help="Leave bitcoinds and test.* datadir on exit or error") - parser.add_option("--srcdir", dest="srcdir", default="../../src", - help="Source directory containing bitcoind/bitcoin-cli (default: %default%)") - parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"), - help="Root directory for datadirs") - (options, args) = parser.parse_args() - - os.environ['PATH'] = options.srcdir+":"+os.environ['PATH'] - - check_json_precision() - - success = False - nodes = [] - try: - print("Initializing test directory "+options.tmpdir) - if not os.path.isdir(options.tmpdir): - os.makedirs(options.tmpdir) - initialize_chain(options.tmpdir) - - run_test(options.tmpdir) - - success = True - - except AssertionError as e: - print("Assertion failed: "+e.message) - except Exception as e: - print("Unexpected exception caught during testing: "+str(e)) - traceback.print_tb(sys.exc_info()[2]) - - if not options.nocleanup: - print("Cleaning up") - wait_bitcoinds() - shutil.rmtree(options.tmpdir) + def setup_nodes(self): + pass - if success: - print("Tests successful") - sys.exit(0) - else: - print("Failed") - sys.exit(1) + def run_bind_test(self, allow_ips, connect_to, addresses, expected): + ''' + Start a node with requested rpcallowip and rpcbind parameters, + then try to connect, and check if the set of bound addresses + matches the expected set. + ''' + expected = [(addr_to_hex(addr), port) for (addr, port) in expected] + base_args = ['-disablewallet', '-nolisten'] + if allow_ips: + base_args += ['-rpcallowip=' + x for x in allow_ips] + binds = ['-rpcbind='+addr for addr in addresses] + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [base_args + binds], connect_to) + try: + pid = bitcoind_processes[0].pid + assert_equal(set(get_bind_addrs(pid)), set(expected)) + finally: + stop_nodes(self.nodes) + wait_bitcoinds() + + def run_allowip_test(self, allow_ips, rpchost, rpcport): + ''' + Start a node with rpcwallow IP, and request getinfo + at a non-localhost IP. + ''' + base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips] + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, [base_args]) + try: + # connect to node through non-loopback interface + node = get_rpc_proxy(rpc_url(0, "%s:%d" % (rpchost, rpcport)), 0) + node.getinfo() + finally: + node = None # make sure connection will be garbage collected and closed + stop_nodes(self.nodes) + wait_bitcoinds() + + def run_test(self): + # due to OS-specific network stats queries, this test works only on Linux + assert(sys.platform.startswith('linux')) + # find the first non-loopback interface for testing + non_loopback_ip = None + for name,ip in all_interfaces(): + if ip != '127.0.0.1': + non_loopback_ip = ip + break + if non_loopback_ip is None: + assert(not 'This test requires at least one non-loopback IPv4 interface') + print("Using interface %s for testing" % non_loopback_ip) + + defaultport = rpc_port(0) + + # check default without rpcallowip (IPv4 and IPv6 localhost) + self.run_bind_test(None, '127.0.0.1', [], + [('127.0.0.1', defaultport), ('::1', defaultport)]) + # check default with rpcallowip (IPv6 any) + self.run_bind_test(['127.0.0.1'], '127.0.0.1', [], + [('::0', defaultport)]) + # check only IPv4 localhost (explicit) + self.run_bind_test(['127.0.0.1'], '127.0.0.1', ['127.0.0.1'], + [('127.0.0.1', defaultport)]) + # check only IPv4 localhost (explicit) with alternative port + self.run_bind_test(['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171'], + [('127.0.0.1', 32171)]) + # check only IPv4 localhost (explicit) with multiple alternative ports on same host + self.run_bind_test(['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171', '127.0.0.1:32172'], + [('127.0.0.1', 32171), ('127.0.0.1', 32172)]) + # check only IPv6 localhost (explicit) + self.run_bind_test(['[::1]'], '[::1]', ['[::1]'], + [('::1', defaultport)]) + # check both IPv4 and IPv6 localhost (explicit) + self.run_bind_test(['127.0.0.1'], '127.0.0.1', ['127.0.0.1', '[::1]'], + [('127.0.0.1', defaultport), ('::1', defaultport)]) + # check only non-loopback interface + self.run_bind_test([non_loopback_ip], non_loopback_ip, [non_loopback_ip], + [(non_loopback_ip, defaultport)]) + + # Check that with invalid rpcallowip, we are denied + self.run_allowip_test([non_loopback_ip], non_loopback_ip, defaultport) + try: + self.run_allowip_test(['1.1.1.1'], non_loopback_ip, defaultport) + assert(not 'Connection not denied by rpcallowip as expected') + except JSONRPCException: + pass if __name__ == '__main__': - main() + RPCBindTest ().main () diff --git a/qa/rpc-tests/segwit.py b/qa/rpc-tests/segwit.py index d4c9a8afed..097e119f32 100755 --- a/qa/rpc-tests/segwit.py +++ b/qa/rpc-tests/segwit.py @@ -69,6 +69,11 @@ def getutxo(txid): utxo["txid"] = txid return utxo +def find_unspent(node, min_value): + for utxo in node.listunspent(): + if utxo['amount'] >= min_value: + return utxo + class SegWitTest(BitcoinTestFramework): def setup_chain(self): @@ -117,8 +122,21 @@ class SegWitTest(BitcoinTestFramework): sync_blocks(self.nodes) def run_test(self): - self.nodes[0].generate(160) #block 160 - + self.nodes[0].generate(161) #block 161 + + print("Verify sigops are counted in GBT with pre-BIP141 rules before the fork") + txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) + tmpl = self.nodes[0].getblocktemplate({}) + assert(tmpl['sigoplimit'] == 20000) + assert(tmpl['transactions'][0]['hash'] == txid) + assert(tmpl['transactions'][0]['sigops'] == 2) + tmpl = self.nodes[0].getblocktemplate({'rules':['segwit']}) + assert(tmpl['sigoplimit'] == 20000) + assert(tmpl['transactions'][0]['hash'] == txid) + assert(tmpl['transactions'][0]['sigops'] == 2) + self.nodes[0].generate(1) #block 162 + + balance_presetup = self.nodes[0].getbalance() self.pubkey = [] p2sh_ids = [] # p2sh_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE embedded in p2sh wit_ids = [] # wit_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE via bare witness @@ -137,18 +155,18 @@ class SegWitTest(BitcoinTestFramework): for i in range(5): for n in range(3): for v in range(2): - wit_ids[n][v].append(send_to_witness(v, self.nodes[0], self.nodes[0].listunspent()[0], self.pubkey[n], False, Decimal("49.999"))) - p2sh_ids[n][v].append(send_to_witness(v, self.nodes[0], self.nodes[0].listunspent()[0], self.pubkey[n], True, Decimal("49.999"))) + wit_ids[n][v].append(send_to_witness(v, self.nodes[0], find_unspent(self.nodes[0], 50), self.pubkey[n], False, Decimal("49.999"))) + p2sh_ids[n][v].append(send_to_witness(v, self.nodes[0], find_unspent(self.nodes[0], 50), self.pubkey[n], True, Decimal("49.999"))) - self.nodes[0].generate(1) #block 161 + self.nodes[0].generate(1) #block 163 sync_blocks(self.nodes) # Make sure all nodes recognize the transactions as theirs - assert_equal(self.nodes[0].getbalance(), 60*50 - 60*50 + 20*Decimal("49.999") + 50) + assert_equal(self.nodes[0].getbalance(), balance_presetup - 60*50 + 20*Decimal("49.999") + 50) assert_equal(self.nodes[1].getbalance(), 20*Decimal("49.999")) assert_equal(self.nodes[2].getbalance(), 20*Decimal("49.999")) - self.nodes[0].generate(262) #block 423 + self.nodes[0].generate(260) #block 423 sync_blocks(self.nodes) print("Verify default node can't accept any witness format txs before fork") @@ -205,5 +223,25 @@ class SegWitTest(BitcoinTestFramework): self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], True) #block 434 self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], True) #block 435 + print("Verify sigops are counted in GBT with BIP141 rules after the fork") + txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) + tmpl = self.nodes[0].getblocktemplate({'rules':['segwit']}) + assert(tmpl['sigoplimit'] == 80000) + assert(tmpl['transactions'][0]['txid'] == txid) + assert(tmpl['transactions'][0]['sigops'] == 8) + + print("Verify non-segwit miners get a valid GBT response after the fork") + send_to_witness(1, self.nodes[0], find_unspent(self.nodes[0], 50), self.pubkey[0], False, Decimal("49.998")) + try: + tmpl = self.nodes[0].getblocktemplate({}) + assert(len(tmpl['transactions']) == 1) # Doesn't include witness tx + assert(tmpl['sigoplimit'] == 20000) + assert(tmpl['transactions'][0]['hash'] == txid) + assert(tmpl['transactions'][0]['sigops'] == 2) + assert(('!segwit' in tmpl['rules']) or ('segwit' not in tmpl['rules'])) + except JSONRPCException: + # This is an acceptable outcome + pass + if __name__ == '__main__': SegWitTest().main() diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index cdd5292cd6..caffab3535 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -36,9 +36,10 @@ from threading import RLock from threading import Thread import logging import copy +from test_framework.siphash import siphash256 BIP0031_VERSION = 60000 -MY_VERSION = 60001 # past bip-31 for ping/pong +MY_VERSION = 70014 # past bip-31 for ping/pong MY_SUBVERSION = b"/python-mininode-tester:0.0.3/" MAX_INV_SZ = 50000 @@ -52,7 +53,7 @@ NODE_BLOOM = (1 << 2) NODE_WITNESS = (1 << 3) # Keep our own socket map for asyncore, so that we can track disconnects -# ourselves (to workaround an issue with closing an asyncore socket when +# ourselves (to workaround an issue with closing an asyncore socket when # using select) mininode_socket_map = dict() @@ -74,8 +75,19 @@ def ripemd160(s): def hash256(s): return sha256(sha256(s)) +def ser_compact_size(l): + r = b"" + if l < 253: + r = struct.pack("B", l) + elif l < 0x10000: + r = struct.pack("<BH", 253, l) + elif l < 0x100000000: + r = struct.pack("<BI", 254, l) + else: + r = struct.pack("<BQ", 255, l) + return r -def deser_string(f): +def deser_compact_size(f): nit = struct.unpack("<B", f.read(1))[0] if nit == 253: nit = struct.unpack("<H", f.read(2))[0] @@ -83,16 +95,14 @@ def deser_string(f): nit = struct.unpack("<I", f.read(4))[0] elif nit == 255: nit = struct.unpack("<Q", f.read(8))[0] + return nit + +def deser_string(f): + nit = deser_compact_size(f) return f.read(nit) def ser_string(s): - if len(s) < 253: - return struct.pack("B", len(s)) + s - elif len(s) < 0x10000: - return struct.pack("<BH", 253, len(s)) + s - elif len(s) < 0x100000000: - return struct.pack("<BI", 254, len(s)) + s - return struct.pack("<BQ", 255, len(s)) + s + return ser_compact_size(len(s)) + s def deser_uint256(f): r = 0 @@ -125,13 +135,7 @@ def uint256_from_compact(c): def deser_vector(f, c): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = c() @@ -144,15 +148,7 @@ def deser_vector(f, c): # entries in the vector (we use this for serializing the vector of transactions # for a witness block). def ser_vector(l, ser_function_name=None): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for i in l: if ser_function_name: r += getattr(i, ser_function_name)() @@ -162,13 +158,7 @@ def ser_vector(l, ser_function_name=None): def deser_uint256_vector(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = deser_uint256(f) @@ -177,28 +167,14 @@ def deser_uint256_vector(f): def ser_uint256_vector(l): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for i in l: r += ser_uint256(i) return r def deser_string_vector(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = deser_string(f) @@ -207,28 +183,14 @@ def deser_string_vector(f): def ser_string_vector(l): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for sv in l: r += ser_string(sv) return r def deser_int_vector(f): - nit = struct.unpack("<B", f.read(1))[0] - if nit == 253: - nit = struct.unpack("<H", f.read(2))[0] - elif nit == 254: - nit = struct.unpack("<I", f.read(4))[0] - elif nit == 255: - nit = struct.unpack("<Q", f.read(8))[0] + nit = deser_compact_size(f) r = [] for i in range(nit): t = struct.unpack("<i", f.read(4))[0] @@ -237,15 +199,7 @@ def deser_int_vector(f): def ser_int_vector(l): - r = b"" - if len(l) < 253: - r = struct.pack("B", len(l)) - elif len(l) < 0x10000: - r = struct.pack("<BH", 253, len(l)) - elif len(l) < 0x100000000: - r = struct.pack("<BI", 254, len(l)) - else: - r = struct.pack("<BQ", 255, len(l)) + r = ser_compact_size(len(l)) for i in l: r += struct.pack("<i", i) return r @@ -294,7 +248,8 @@ class CInv(object): 1: "TX", 2: "Block", 1|MSG_WITNESS_FLAG: "WitnessTx", - 2|MSG_WITNESS_FLAG : "WitnessBlock" + 2|MSG_WITNESS_FLAG : "WitnessBlock", + 4: "CompactBlock" } def __init__(self, t=0, h=0): @@ -781,6 +736,187 @@ class CAlert(object): % (len(self.vchMsg), len(self.vchSig)) +class PrefilledTransaction(object): + def __init__(self, index=0, tx = None): + self.index = index + self.tx = tx + + def deserialize(self, f): + self.index = deser_compact_size(f) + self.tx = CTransaction() + self.tx.deserialize(f) + + def serialize(self, with_witness=False): + r = b"" + r += ser_compact_size(self.index) + if with_witness: + r += self.tx.serialize_with_witness() + else: + r += self.tx.serialize_without_witness() + return r + + def __repr__(self): + return "PrefilledTransaction(index=%d, tx=%s)" % (self.index, repr(self.tx)) + +# This is what we send on the wire, in a cmpctblock message. +class P2PHeaderAndShortIDs(object): + def __init__(self): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids_length = 0 + self.shortids = [] + self.prefilled_txn_length = 0 + self.prefilled_txn = [] + + def deserialize(self, f): + self.header.deserialize(f) + self.nonce = struct.unpack("<Q", f.read(8))[0] + self.shortids_length = deser_compact_size(f) + for i in range(self.shortids_length): + # shortids are defined to be 6 bytes in the spec, so append + # two zero bytes and read it in as an 8-byte number + self.shortids.append(struct.unpack("<Q", f.read(6) + b'\x00\x00')[0]) + self.prefilled_txn = deser_vector(f, PrefilledTransaction) + self.prefilled_txn_length = len(self.prefilled_txn) + + def serialize(self, with_witness=False): + r = b"" + r += self.header.serialize() + r += struct.pack("<Q", self.nonce) + r += ser_compact_size(self.shortids_length) + for x in self.shortids: + # We only want the first 6 bytes + r += struct.pack("<Q", x)[0:6] + r += ser_vector(self.prefilled_txn) + return r + + def __repr__(self): + return "P2PHeaderAndShortIDs(header=%s, nonce=%d, shortids_length=%d, shortids=%s, prefilled_txn_length=%d, prefilledtxn=%s" % (repr(self.header), self.nonce, self.shortids_length, repr(self.shortids), self.prefilled_txn_length, repr(self.prefilled_txn)) + + +# Calculate the BIP 152-compact blocks shortid for a given transaction hash +def calculate_shortid(k0, k1, tx_hash): + expected_shortid = siphash256(k0, k1, tx_hash) + expected_shortid &= 0x0000ffffffffffff + return expected_shortid + +# This version gets rid of the array lengths, and reinterprets the differential +# encoding into indices that can be used for lookup. +class HeaderAndShortIDs(object): + def __init__(self, p2pheaders_and_shortids = None): + self.header = CBlockHeader() + self.nonce = 0 + self.shortids = [] + self.prefilled_txn = [] + + if p2pheaders_and_shortids != None: + self.header = p2pheaders_and_shortids.header + self.nonce = p2pheaders_and_shortids.nonce + self.shortids = p2pheaders_and_shortids.shortids + last_index = -1 + for x in p2pheaders_and_shortids.prefilled_txn: + self.prefilled_txn.append(PrefilledTransaction(x.index + last_index + 1, x.tx)) + last_index = self.prefilled_txn[-1].index + + def to_p2p(self): + ret = P2PHeaderAndShortIDs() + ret.header = self.header + ret.nonce = self.nonce + ret.shortids_length = len(self.shortids) + ret.shortids = self.shortids + ret.prefilled_txn_length = len(self.prefilled_txn) + ret.prefilled_txn = [] + last_index = -1 + for x in self.prefilled_txn: + ret.prefilled_txn.append(PrefilledTransaction(x.index - last_index - 1, x.tx)) + last_index = x.index + return ret + + def get_siphash_keys(self): + header_nonce = self.header.serialize() + header_nonce += struct.pack("<Q", self.nonce) + hash_header_nonce_as_str = sha256(header_nonce) + key0 = struct.unpack("<Q", hash_header_nonce_as_str[0:8])[0] + key1 = struct.unpack("<Q", hash_header_nonce_as_str[8:16])[0] + return [ key0, key1 ] + + def initialize_from_block(self, block, nonce=0, prefill_list = [0]): + self.header = CBlockHeader(block) + self.nonce = nonce + self.prefilled_txn = [ PrefilledTransaction(i, block.vtx[i]) for i in prefill_list ] + self.shortids = [] + [k0, k1] = self.get_siphash_keys() + for i in range(len(block.vtx)): + if i not in prefill_list: + self.shortids.append(calculate_shortid(k0, k1, block.vtx[i].sha256)) + + def __repr__(self): + return "HeaderAndShortIDs(header=%s, nonce=%d, shortids=%s, prefilledtxn=%s" % (repr(self.header), self.nonce, repr(self.shortids), repr(self.prefilled_txn)) + + +class BlockTransactionsRequest(object): + + def __init__(self, blockhash=0, indexes = None): + self.blockhash = blockhash + self.indexes = indexes if indexes != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + indexes_length = deser_compact_size(f) + for i in range(indexes_length): + self.indexes.append(deser_compact_size(f)) + + def serialize(self): + r = b"" + r += ser_uint256(self.blockhash) + r += ser_compact_size(len(self.indexes)) + for x in self.indexes: + r += ser_compact_size(x) + return r + + # helper to set the differentially encoded indexes from absolute ones + def from_absolute(self, absolute_indexes): + self.indexes = [] + last_index = -1 + for x in absolute_indexes: + self.indexes.append(x-last_index-1) + last_index = x + + def to_absolute(self): + absolute_indexes = [] + last_index = -1 + for x in self.indexes: + absolute_indexes.append(x+last_index+1) + last_index = absolute_indexes[-1] + return absolute_indexes + + def __repr__(self): + return "BlockTransactionsRequest(hash=%064x indexes=%s)" % (self.blockhash, repr(self.indexes)) + + +class BlockTransactions(object): + + def __init__(self, blockhash=0, transactions = None): + self.blockhash = blockhash + self.transactions = transactions if transactions != None else [] + + def deserialize(self, f): + self.blockhash = deser_uint256(f) + self.transactions = deser_vector(f, CTransaction) + + def serialize(self, with_witness=False): + r = b"" + r += ser_uint256(self.blockhash) + if with_witness: + r += ser_vector(self.transactions, "serialize_with_witness") + else: + r += ser_vector(self.transactions) + return r + + def __repr__(self): + return "BlockTransactions(hash=%064x transactions=%s)" % (self.blockhash, repr(self.transactions)) + + # Objects that correspond to messages on the wire class msg_version(object): command = b"version" @@ -1215,6 +1351,79 @@ class msg_feefilter(object): def __repr__(self): return "msg_feefilter(feerate=%08x)" % self.feerate +class msg_sendcmpct(object): + command = b"sendcmpct" + + def __init__(self): + self.announce = False + self.version = 1 + + def deserialize(self, f): + self.announce = struct.unpack("<?", f.read(1))[0] + self.version = struct.unpack("<Q", f.read(8))[0] + + def serialize(self): + r = b"" + r += struct.pack("<?", self.announce) + r += struct.pack("<Q", self.version) + return r + + def __repr__(self): + return "msg_sendcmpct(announce=%s, version=%lu)" % (self.announce, self.version) + +class msg_cmpctblock(object): + command = b"cmpctblock" + + def __init__(self, header_and_shortids = None): + self.header_and_shortids = header_and_shortids + + def deserialize(self, f): + self.header_and_shortids = P2PHeaderAndShortIDs() + self.header_and_shortids.deserialize(f) + + def serialize(self): + r = b"" + r += self.header_and_shortids.serialize() + return r + + def __repr__(self): + return "msg_cmpctblock(HeaderAndShortIDs=%s)" % repr(self.header_and_shortids) + +class msg_getblocktxn(object): + command = b"getblocktxn" + + def __init__(self): + self.block_txn_request = None + + def deserialize(self, f): + self.block_txn_request = BlockTransactionsRequest() + self.block_txn_request.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_txn_request.serialize() + return r + + def __repr__(self): + return "msg_getblocktxn(block_txn_request=%s)" % (repr(self.block_txn_request)) + +class msg_blocktxn(object): + command = b"blocktxn" + + def __init__(self): + self.block_transactions = BlockTransactions() + + def deserialize(self, f): + self.block_transactions.deserialize(f) + + def serialize(self): + r = b"" + r += self.block_transactions.serialize() + return r + + def __repr__(self): + return "msg_blocktxn(block_transactions=%s)" % (repr(self.block_transactions)) + # This is what a callback should look like for NodeConn # Reimplement the on_* functions to provide handling for events class NodeConnCB(object): @@ -1295,6 +1504,10 @@ class NodeConnCB(object): def on_pong(self, conn, message): pass def on_feefilter(self, conn, message): pass def on_sendheaders(self, conn, message): pass + def on_sendcmpct(self, conn, message): pass + def on_cmpctblock(self, conn, message): pass + def on_getblocktxn(self, conn, message): pass + def on_blocktxn(self, conn, message): pass # More useful callbacks and functions for NodeConnCB's which have a single NodeConn class SingleNodeConnCB(NodeConnCB): @@ -1311,6 +1524,10 @@ class SingleNodeConnCB(NodeConnCB): def send_message(self, message): self.connection.send_message(message) + def send_and_ping(self, message): + self.send_message(message) + self.sync_with_ping() + def on_pong(self, conn, message): self.last_pong = message @@ -1344,7 +1561,11 @@ class NodeConn(asyncore.dispatcher): b"reject": msg_reject, b"mempool": msg_mempool, b"feefilter": msg_feefilter, - b"sendheaders": msg_sendheaders + b"sendheaders": msg_sendheaders, + b"sendcmpct": msg_sendcmpct, + b"cmpctblock": msg_cmpctblock, + b"getblocktxn": msg_getblocktxn, + b"blocktxn": msg_blocktxn } MAGIC_BYTES = { "mainnet": b"\xf9\xbe\xb4\xd9", # mainnet diff --git a/qa/rpc-tests/test_framework/siphash.py b/qa/rpc-tests/test_framework/siphash.py new file mode 100644 index 0000000000..9c0574bd93 --- /dev/null +++ b/qa/rpc-tests/test_framework/siphash.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# siphash.py - Specialized SipHash-2-4 implementations +# +# This implements SipHash-2-4 for 256-bit integers. + +def rotl64(n, b): + return n >> (64 - b) | (n & ((1 << (64 - b)) - 1)) << b + +def siphash_round(v0, v1, v2, v3): + v0 = (v0 + v1) & ((1 << 64) - 1) + v1 = rotl64(v1, 13) + v1 ^= v0 + v0 = rotl64(v0, 32) + v2 = (v2 + v3) & ((1 << 64) - 1) + v3 = rotl64(v3, 16) + v3 ^= v2 + v0 = (v0 + v3) & ((1 << 64) - 1) + v3 = rotl64(v3, 21) + v3 ^= v0 + v2 = (v2 + v1) & ((1 << 64) - 1) + v1 = rotl64(v1, 17) + v1 ^= v2 + v2 = rotl64(v2, 32) + return (v0, v1, v2, v3) + +def siphash256(k0, k1, h): + n0 = h & ((1 << 64) - 1) + n1 = (h >> 64) & ((1 << 64) - 1) + n2 = (h >> 128) & ((1 << 64) - 1) + n3 = (h >> 192) & ((1 << 64) - 1) + v0 = 0x736f6d6570736575 ^ k0 + v1 = 0x646f72616e646f6d ^ k1 + v2 = 0x6c7967656e657261 ^ k0 + v3 = 0x7465646279746573 ^ k1 ^ n0 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n0 + v3 ^= n1 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n1 + v3 ^= n2 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n2 + v3 ^= n3 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= n3 + v3 ^= 0x2000000000000000 + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0 ^= 0x2000000000000000 + v2 ^= 0xFF + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + v0, v1, v2, v3 = siphash_round(v0, v1, v2, v3) + return v0 ^ v1 ^ v2 ^ v3 diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 32fe79efc3..8aa34265c5 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -171,7 +171,15 @@ def rpc_auth_pair(n): def rpc_url(i, rpchost=None): rpc_u, rpc_p = rpc_auth_pair(i) - return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, rpchost or '127.0.0.1', rpc_port(i)) + host = '127.0.0.1' + port = rpc_port(i) + if rpchost: + parts = rpchost.split(':') + if len(parts) == 2: + host, port = parts + else: + host = rpchost + return "http://%s:%s@%s:%d" % (rpc_u, rpc_p, host, int(port)) def wait_for_bitcoind_start(process, url, i): ''' diff --git a/qa/rpc-tests/wallet-dump.py b/qa/rpc-tests/wallet-dump.py new file mode 100755 index 0000000000..6028d2c20b --- /dev/null +++ b/qa/rpc-tests/wallet-dump.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import (start_nodes, start_node, assert_equal, bitcoind_processes) + + +def read_dump(file_name, addrs, hd_master_addr_old): + """ + Read the given dump, count the addrs that match, count change and reserve. + Also check that the old hd_master is inactive + """ + with open(file_name) as inputfile: + found_addr = 0 + found_addr_chg = 0 + found_addr_rsv = 0 + hd_master_addr_ret = None + for line in inputfile: + # only read non comment lines + if line[0] != "#" and len(line) > 10: + # split out some data + key_label, comment = line.split("#") + # key = key_label.split(" ")[0] + keytype = key_label.split(" ")[2] + if len(comment) > 1: + addr_keypath = comment.split(" addr=")[1] + addr = addr_keypath.split(" ")[0] + keypath = None + if keytype == "inactivehdmaster=1": + # ensure the old master is still available + assert(hd_master_addr_old == addr) + elif keytype == "hdmaster=1": + # ensure we have generated a new hd master key + assert(hd_master_addr_old != addr) + hd_master_addr_ret = addr + else: + keypath = addr_keypath.rstrip().split("hdkeypath=")[1] + + # count key types + for addrObj in addrs: + if addrObj['address'] == addr and addrObj['hdkeypath'] == keypath and keytype == "label=": + found_addr += 1 + break + elif keytype == "change=1": + found_addr_chg += 1 + break + elif keytype == "reserve=1": + found_addr_rsv += 1 + break + return found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_ret + + +class WalletDumpTest(BitcoinTestFramework): + + def __init__(self): + super().__init__() + self.setup_clean_chain = False + self.num_nodes = 1 + self.extra_args = [["-keypool=90"]] + + def setup_network(self, split=False): + self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, self.extra_args) + + def run_test (self): + tmpdir = self.options.tmpdir + + # generate 20 addresses to compare against the dump + test_addr_count = 20 + addrs = [] + for i in range(0,test_addr_count): + addr = self.nodes[0].getnewaddress() + vaddr= self.nodes[0].validateaddress(addr) #required to get hd keypath + addrs.append(vaddr) + # Should be a no-op: + self.nodes[0].keypoolrefill() + + # dump unencrypted wallet + self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.unencrypted.dump") + + found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_unenc = \ + read_dump(tmpdir + "/node0/wallet.unencrypted.dump", addrs, None) + assert_equal(found_addr, test_addr_count) # all keys must be in the dump + assert_equal(found_addr_chg, 50) # 50 blocks where mined + assert_equal(found_addr_rsv, 90 + 1) # keypool size (TODO: fix off-by-one) + + #encrypt wallet, restart, unlock and dump + self.nodes[0].encryptwallet('test') + bitcoind_processes[0].wait() + self.nodes[0] = start_node(0, self.options.tmpdir, self.extra_args[0]) + self.nodes[0].walletpassphrase('test', 10) + # Should be a no-op: + self.nodes[0].keypoolrefill() + self.nodes[0].dumpwallet(tmpdir + "/node0/wallet.encrypted.dump") + + found_addr, found_addr_chg, found_addr_rsv, hd_master_addr_enc = \ + read_dump(tmpdir + "/node0/wallet.encrypted.dump", addrs, hd_master_addr_unenc) + assert_equal(found_addr, test_addr_count) + assert_equal(found_addr_chg, 90 + 1 + 50) # old reserve keys are marked as change now + assert_equal(found_addr_rsv, 90 + 1) # keypool size (TODO: fix off-by-one) + +if __name__ == '__main__': + WalletDumpTest().main () diff --git a/qa/rpc-tests/wallet-hd.py b/qa/rpc-tests/wallet-hd.py index c738ee2207..c11da1e9a9 100755 --- a/qa/rpc-tests/wallet-hd.py +++ b/qa/rpc-tests/wallet-hd.py @@ -31,7 +31,7 @@ class WalletHDTest(BitcoinTestFramework): tmpdir = self.options.tmpdir # Make sure we use hd, keep masterkeyid - masterkeyid = self.nodes[1].getwalletinfo()['masterkeyid'] + masterkeyid = self.nodes[1].getwalletinfo()['hdmasterkeyid'] assert_equal(len(masterkeyid), 40) # Import a non-HD private key in the HD wallet diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 86bef1e105..ea6e3aada2 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -71,11 +71,10 @@ public: CMainParams() { strNetworkID = "main"; consensus.nSubsidyHalvingInterval = 210000; - consensus.nMajorityEnforceBlockUpgrade = 750; - consensus.nMajorityRejectBlockOutdated = 950; - consensus.nMajorityWindow = 1000; consensus.BIP34Height = 227931; consensus.BIP34Hash = uint256S("0x000000000000024b89b42a942fe0d9fea3bb44ab7bd1b19115dd6a759c0808b8"); + consensus.BIP65Height = 388381; // 000000000000000004c2b624ed5d7756c508d90fd0da2c7c679febfa6c4735f0 + consensus.BIP66Height = 363725; // 00000000000000000379eaa19dce8c9b722d46ae6a57c2f1a988119488b50931 consensus.powLimit = uint256S("00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; @@ -167,11 +166,10 @@ public: CTestNetParams() { strNetworkID = "test"; consensus.nSubsidyHalvingInterval = 210000; - consensus.nMajorityEnforceBlockUpgrade = 51; - consensus.nMajorityRejectBlockOutdated = 75; - consensus.nMajorityWindow = 100; consensus.BIP34Height = 21111; consensus.BIP34Hash = uint256S("0x0000000023b3a96d3484e5abb3755c413e7d41500f8e2a5c3f0dd01299cd8ef8"); + consensus.BIP65Height = 581885; // 00000000007f6655f22f98e72ed80d8b06dc761d5da09df0fa1dc4be4f861eb6 + consensus.BIP66Height = 330776; // 000000002104c8c45e99a8853285a3b592602a3ccde2b832481da85e9e4ba182 consensus.powLimit = uint256S("00000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; @@ -247,11 +245,10 @@ public: CRegTestParams() { strNetworkID = "regtest"; consensus.nSubsidyHalvingInterval = 150; - consensus.nMajorityEnforceBlockUpgrade = 750; - consensus.nMajorityRejectBlockOutdated = 950; - consensus.nMajorityWindow = 1000; - consensus.BIP34Height = -1; // BIP34 has not necessarily activated on regtest + consensus.BIP34Height = 100000000; // BIP34 has not activated on regtest (far in the future so block v1 are not rejected in tests) consensus.BIP34Hash = uint256(); + consensus.BIP65Height = 1351; // BIP65 activated on regtest (Used in rpc activation tests) + consensus.BIP66Height = 1251; // BIP66 activated on regtest (Used in rpc activation tests) consensus.powLimit = uint256S("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks consensus.nPowTargetSpacing = 10 * 60; @@ -303,6 +300,12 @@ public: base58Prefixes[EXT_PUBLIC_KEY] = boost::assign::list_of(0x04)(0x35)(0x87)(0xCF).convert_to_container<std::vector<unsigned char> >(); base58Prefixes[EXT_SECRET_KEY] = boost::assign::list_of(0x04)(0x35)(0x83)(0x94).convert_to_container<std::vector<unsigned char> >(); } + + void UpdateBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) + { + consensus.vDeployments[d].nStartTime = nStartTime; + consensus.vDeployments[d].nTimeout = nTimeout; + } }; static CRegTestParams regTestParams; @@ -330,4 +333,9 @@ void SelectParams(const std::string& network) SelectBaseParams(network); pCurrentParams = &Params(network); } + +void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout) +{ + regTestParams.UpdateBIP9Parameters(d, nStartTime, nTimeout); +} diff --git a/src/chainparams.h b/src/chainparams.h index 638893e9ad..0c3820b7c6 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -112,4 +112,9 @@ CChainParams& Params(const std::string& chain); */ void SelectParams(const std::string& chain); +/** + * Allows modifying the BIP9 regtest parameters. + */ +void UpdateRegtestBIP9Parameters(Consensus::DeploymentPos d, int64_t nStartTime, int64_t nTimeout); + #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/clientversion.h b/src/clientversion.h index 47263d5344..53ad460346 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -15,7 +15,7 @@ //! These need to be macros, as clientversion.cpp's and bitcoin*-res.rc's voodoo requires it #define CLIENT_VERSION_MAJOR 0 -#define CLIENT_VERSION_MINOR 12 +#define CLIENT_VERSION_MINOR 13 #define CLIENT_VERSION_REVISION 99 #define CLIENT_VERSION_BUILD 0 diff --git a/src/consensus/consensus.h b/src/consensus/consensus.h index 81f40593b2..6908565866 100644 --- a/src/consensus/consensus.h +++ b/src/consensus/consensus.h @@ -10,8 +10,8 @@ /** The maximum allowed size for a serialized block, in bytes (only for buffer size limits) */ static const unsigned int MAX_BLOCK_SERIALIZED_SIZE = 4000000; -/** The maximum allowed cost for a block, see BIP 141 (network rule) */ -static const unsigned int MAX_BLOCK_COST = 4000000; +/** The maximum allowed weight for a block, see BIP 141 (network rule) */ +static const unsigned int MAX_BLOCK_WEIGHT = 4000000; /** The maximum allowed size for a block excluding witness data, in bytes (network rule) */ static const unsigned int MAX_BLOCK_BASE_SIZE = 1000000; /** The maximum allowed number of signature check operations in a block (network rule) */ diff --git a/src/consensus/params.h b/src/consensus/params.h index 822ec87d69..5b2f49184f 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -39,13 +39,13 @@ struct BIP9Deployment { struct Params { uint256 hashGenesisBlock; int nSubsidyHalvingInterval; - /** Used to check majorities for block version upgrade */ - int nMajorityEnforceBlockUpgrade; - int nMajorityRejectBlockOutdated; - int nMajorityWindow; /** Block height and hash at which BIP34 becomes active */ int BIP34Height; uint256 BIP34Hash; + /** Block height at which BIP65 becomes active */ + int BIP65Height; + /** Block height at which BIP66 becomes active */ + int BIP66Height; /** * Minimum blocks including miner confirmation of the total of 2016 blocks in a retargetting period, * (nPowTargetTimespan / nPowTargetSpacing) which is also used for BIP9 deployments. diff --git a/src/dbwrapper.h b/src/dbwrapper.h index a0779d3ab9..47bdb31b5b 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -52,9 +52,9 @@ private: public: /** - * @param[in] parent CDBWrapper that this batch is to be submitted to + * @param[in] _parent CDBWrapper that this batch is to be submitted to */ - CDBBatch(const CDBWrapper &parent) : parent(parent) { }; + CDBBatch(const CDBWrapper &_parent) : parent(_parent) { }; template <typename K, typename V> void Write(const K& key, const V& value) @@ -94,11 +94,11 @@ private: public: /** - * @param[in] parent Parent CDBWrapper instance. - * @param[in] piterIn The original leveldb iterator. + * @param[in] _parent Parent CDBWrapper instance. + * @param[in] _piter The original leveldb iterator. */ - CDBIterator(const CDBWrapper &parent, leveldb::Iterator *piterIn) : - parent(parent), piter(piterIn) { }; + CDBIterator(const CDBWrapper &_parent, leveldb::Iterator *_piter) : + parent(_parent), piter(_piter) { }; ~CDBIterator(); bool Valid(); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 812940eaf9..be7a6a1dde 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -19,6 +19,7 @@ #include <sys/types.h> #include <sys/stat.h> #include <signal.h> +#include <future> #include <event2/event.h> #include <event2/http.h> @@ -34,9 +35,6 @@ #endif #endif -#include <boost/algorithm/string/case_conv.hpp> // for to_lower() -#include <boost/foreach.hpp> - /** Maximum size of http request (request line + headers) */ static const size_t MAX_HEADERS_SIZE = 8192; @@ -68,8 +66,8 @@ class WorkQueue { private: /** Mutex protects entire object */ - CWaitableCriticalSection cs; - CConditionVariable cond; + std::mutex cs; + std::condition_variable cond; std::deque<std::unique_ptr<WorkItem>> queue; bool running; size_t maxDepth; @@ -82,12 +80,12 @@ private: WorkQueue &wq; ThreadCounter(WorkQueue &w): wq(w) { - boost::lock_guard<boost::mutex> lock(wq.cs); + std::lock_guard<std::mutex> lock(wq.cs); wq.numThreads += 1; } ~ThreadCounter() { - boost::lock_guard<boost::mutex> lock(wq.cs); + std::lock_guard<std::mutex> lock(wq.cs); wq.numThreads -= 1; wq.cond.notify_all(); } @@ -108,7 +106,7 @@ public: /** Enqueue a work item */ bool Enqueue(WorkItem* item) { - boost::unique_lock<boost::mutex> lock(cs); + std::unique_lock<std::mutex> lock(cs); if (queue.size() >= maxDepth) { return false; } @@ -123,7 +121,7 @@ public: while (running) { std::unique_ptr<WorkItem> i; { - boost::unique_lock<boost::mutex> lock(cs); + std::unique_lock<std::mutex> lock(cs); while (running && queue.empty()) cond.wait(lock); if (!running) @@ -137,14 +135,14 @@ public: /** Interrupt and exit loops */ void Interrupt() { - boost::unique_lock<boost::mutex> lock(cs); + std::unique_lock<std::mutex> lock(cs); running = false; cond.notify_all(); } /** Wait for worker threads to exit */ void WaitExit() { - boost::unique_lock<boost::mutex> lock(cs); + std::unique_lock<std::mutex> lock(cs); while (numThreads > 0) cond.wait(lock); } @@ -152,7 +150,7 @@ public: /** Return current depth of queue */ size_t Depth() { - boost::unique_lock<boost::mutex> lock(cs); + std::unique_lock<std::mutex> lock(cs); return queue.size(); } }; @@ -189,7 +187,7 @@ static bool ClientAllowed(const CNetAddr& netaddr) { if (!netaddr.IsValid()) return false; - BOOST_FOREACH (const CSubNet& subnet, rpc_allow_subnets) + for(const CSubNet& subnet : rpc_allow_subnets) if (subnet.Match(netaddr)) return true; return false; @@ -203,7 +201,7 @@ static bool InitHTTPAllowList() rpc_allow_subnets.push_back(CSubNet("::1")); // always allow IPv6 localhost if (mapMultiArgs.count("-rpcallowip")) { const std::vector<std::string>& vAllow = mapMultiArgs["-rpcallowip"]; - BOOST_FOREACH (std::string strAllow, vAllow) { + for (std::string strAllow : vAllow) { CSubNet subnet(strAllow); if (!subnet.IsValid()) { uiInterface.ThreadSafeMessageBox( @@ -215,7 +213,7 @@ static bool InitHTTPAllowList() } } std::string strAllowed; - BOOST_FOREACH (const CSubNet& subnet, rpc_allow_subnets) + for (const CSubNet& subnet : rpc_allow_subnets) strAllowed += subnet.ToString() + " "; LogPrint("http", "Allowing HTTP connections from: %s\n", strAllowed); return true; @@ -302,13 +300,14 @@ static void http_reject_request_cb(struct evhttp_request* req, void*) } /** Event dispatcher thread */ -static void ThreadHTTP(struct event_base* base, struct evhttp* http) +static bool ThreadHTTP(struct event_base* base, struct evhttp* http) { RenameThread("bitcoin-http"); LogPrint("http", "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() LogPrint("http", "Exited http event loop\n"); + return event_base_got_break(base) == 0; } /** Bind HTTP server to specified addresses */ @@ -437,17 +436,22 @@ bool InitHTTPServer() return true; } -boost::thread threadHTTP; +std::thread threadHTTP; +std::future<bool> threadResult; bool StartHTTPServer() { LogPrint("http", "Starting HTTP server\n"); int rpcThreads = std::max((long)GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); - threadHTTP = boost::thread(boost::bind(&ThreadHTTP, eventBase, eventHTTP)); + std::packaged_task<bool(event_base*, evhttp*)> task(ThreadHTTP); + threadResult = task.get_future(); + threadHTTP = std::thread(std::move(task), eventBase, eventHTTP); - for (int i = 0; i < rpcThreads; i++) - boost::thread(boost::bind(&HTTPWorkQueueRun, workQueue)); + for (int i = 0; i < rpcThreads; i++) { + std::thread rpc_worker(HTTPWorkQueueRun, workQueue); + rpc_worker.detach(); + } return true; } @@ -456,7 +460,7 @@ void InterruptHTTPServer() LogPrint("http", "Interrupting HTTP server\n"); if (eventHTTP) { // Unlisten sockets - BOOST_FOREACH (evhttp_bound_socket *socket, boundSockets) { + for (evhttp_bound_socket *socket : boundSockets) { evhttp_del_accept_socket(eventHTTP, socket); } // Reject requests on current connections @@ -482,15 +486,11 @@ void StopHTTPServer() // master that appears to be solved, so in the future that solution // could be used again (if desirable). // (see discussion in https://github.com/bitcoin/bitcoin/pull/6990) -#if BOOST_VERSION >= 105000 - if (!threadHTTP.try_join_for(boost::chrono::milliseconds(2000))) { -#else - if (!threadHTTP.timed_join(boost::posix_time::milliseconds(2000))) { -#endif + if (threadResult.valid() && threadResult.wait_for(std::chrono::milliseconds(2000)) == std::future_status::timeout) { LogPrintf("HTTP event loop did not exit within allotted time, sending loopbreak\n"); event_base_loopbreak(eventBase); - threadHTTP.join(); } + threadHTTP.join(); } if (eventHTTP) { evhttp_free(eventHTTP); @@ -517,7 +517,7 @@ static void httpevent_callback_fn(evutil_socket_t, short, void* data) delete self; } -HTTPEvent::HTTPEvent(struct event_base* base, bool deleteWhenTriggered, const boost::function<void(void)>& handler): +HTTPEvent::HTTPEvent(struct event_base* base, bool deleteWhenTriggered, const std::function<void(void)>& handler): deleteWhenTriggered(deleteWhenTriggered), handler(handler) { ev = event_new(base, -1, 0, httpevent_callback_fn, this); @@ -599,7 +599,7 @@ void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) assert(evb); evbuffer_add(evb, strReply.data(), strReply.size()); HTTPEvent* ev = new HTTPEvent(eventBase, true, - boost::bind(evhttp_send_reply, req, nStatus, (const char*)NULL, (struct evbuffer *)NULL)); + std::bind(evhttp_send_reply, req, nStatus, (const char*)NULL, (struct evbuffer *)NULL)); ev->trigger(0); replySent = true; req = 0; // transferred back to main thread diff --git a/src/httpserver.h b/src/httpserver.h index 20a119cc5c..0e30e666a6 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -7,9 +7,7 @@ #include <string> #include <stdint.h> -#include <boost/thread.hpp> -#include <boost/scoped_ptr.hpp> -#include <boost/function.hpp> +#include <functional> static const int DEFAULT_HTTP_THREADS=4; static const int DEFAULT_HTTP_WORKQUEUE=16; @@ -35,7 +33,7 @@ void InterruptHTTPServer(); void StopHTTPServer(); /** Handler for requests to a certain HTTP path */ -typedef boost::function<void(HTTPRequest* req, const std::string &)> HTTPRequestHandler; +typedef std::function<void(HTTPRequest* req, const std::string &)> HTTPRequestHandler; /** Register handler for prefix. * If multiple handlers match a prefix, the first-registered one will * be invoked. @@ -132,7 +130,7 @@ public: * deleteWhenTriggered deletes this event object after the event is triggered (and the handler called) * handler is the handler to call when the event is triggered. */ - HTTPEvent(struct event_base* base, bool deleteWhenTriggered, const boost::function<void(void)>& handler); + HTTPEvent(struct event_base* base, bool deleteWhenTriggered, const std::function<void(void)>& handler); ~HTTPEvent(); /** Trigger the event. If tv is 0, trigger it immediately. Otherwise trigger it after @@ -141,7 +139,7 @@ public: void trigger(struct timeval* tv); bool deleteWhenTriggered; - boost::function<void(void)> handler; + std::function<void(void)> handler; private: struct event* ev; }; diff --git a/src/indirectmap.h b/src/indirectmap.h index 28e1e8dedd..76da4a6bd5 100644 --- a/src/indirectmap.h +++ b/src/indirectmap.h @@ -1,3 +1,7 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #ifndef BITCOIN_INDIRECTMAP_H #define BITCOIN_INDIRECTMAP_H diff --git a/src/init.cpp b/src/init.cpp index 509bc45f51..4d9de6cfc8 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -410,6 +410,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT)); + strUsage += HelpMessageOpt("-bip9params=deployment:start:end", "Use given start/end times for specified BIP9 deployment (regtest-only)"); } string debugCategories = "addrman, alert, bench, coindb, db, http, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq"; // Don't translate these and qt below if (mode == HMM_BITCOIN_QT) @@ -446,13 +447,13 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageGroup(_("Node relay options:")); if (showDebug) strUsage += HelpMessageOpt("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", !Params(CBaseChainParams::TESTNET).RequireStandard())); - strUsage += HelpMessageOpt("-bytespersigop", strprintf(_("Minimum bytes per sigop in transactions we relay and mine (default: %u)"), DEFAULT_BYTES_PER_SIGOP)); + strUsage += HelpMessageOpt("-bytespersigop", strprintf(_("Equivalent bytes per sigop in transactions for relay and mining (default: %u)"), DEFAULT_BYTES_PER_SIGOP)); strUsage += HelpMessageOpt("-datacarrier", strprintf(_("Relay and mine data carrier transactions (default: %u)"), DEFAULT_ACCEPT_DATACARRIER)); strUsage += HelpMessageOpt("-datacarriersize", strprintf(_("Maximum size of data in data carrier transactions we relay and mine (default: %u)"), MAX_OP_RETURN_RELAY)); strUsage += HelpMessageOpt("-mempoolreplacement", strprintf(_("Enable transaction replacement in the memory pool (default: %u)"), DEFAULT_ENABLE_REPLACEMENT)); strUsage += HelpMessageGroup(_("Block creation options:")); - strUsage += HelpMessageOpt("-blockmaxcost=<n>", strprintf(_("Set maximum BIP141 block cost (default: %d)"), DEFAULT_BLOCK_MAX_COST)); + strUsage += HelpMessageOpt("-blockmaxweight=<n>", strprintf(_("Set maximum BIP141 block weight (default: %d)"), DEFAULT_BLOCK_MAX_WEIGHT)); strUsage += HelpMessageOpt("-blockmaxsize=<n>", strprintf(_("Set maximum block size in bytes (default: %d)"), DEFAULT_BLOCK_MAX_SIZE)); strUsage += HelpMessageOpt("-blockprioritysize=<n>", strprintf(_("Set maximum size of high-priority/low-fee transactions in bytes (default: %d)"), DEFAULT_BLOCK_PRIORITY_SIZE)); if (showDebug) @@ -510,6 +511,21 @@ static void BlockNotifyCallback(bool initialSync, const CBlockIndex *pBlockIndex boost::thread t(runCommand, strCmd); // thread runs free } +static bool fHaveGenesis = false; +static boost::mutex cs_GenesisWait; +static CConditionVariable condvar_GenesisWait; + +static void BlockNotifyGenesisWait(bool, const CBlockIndex *pBlockIndex) +{ + if (pBlockIndex != NULL) { + { + boost::unique_lock<boost::mutex> lock_GenesisWait(cs_GenesisWait); + fHaveGenesis = true; + } + condvar_GenesisWait.notify_all(); + } +} + struct CImportingNow { CImportingNow() { @@ -975,6 +991,41 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) fEnableReplacement = (std::find(vstrReplacementModes.begin(), vstrReplacementModes.end(), "fee") != vstrReplacementModes.end()); } + if (!mapMultiArgs["-bip9params"].empty()) { + // Allow overriding BIP9 parameters for testing + if (!Params().MineBlocksOnDemand()) { + return InitError("BIP9 parameters may only be overridden on regtest."); + } + const vector<string>& deployments = mapMultiArgs["-bip9params"]; + for (auto i : deployments) { + std::vector<std::string> vDeploymentParams; + boost::split(vDeploymentParams, i, boost::is_any_of(":")); + if (vDeploymentParams.size() != 3) { + return InitError("BIP9 parameters malformed, expecting deployment:start:end"); + } + int64_t nStartTime, nTimeout; + if (!ParseInt64(vDeploymentParams[1], &nStartTime)) { + return InitError(strprintf("Invalid nStartTime (%s)", vDeploymentParams[1])); + } + if (!ParseInt64(vDeploymentParams[2], &nTimeout)) { + return InitError(strprintf("Invalid nTimeout (%s)", vDeploymentParams[2])); + } + bool found = false; + for (int j=0; j<(int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) + { + if (vDeploymentParams[0].compare(VersionBitsDeploymentInfo[j].name) == 0) { + UpdateRegtestBIP9Parameters(Consensus::DeploymentPos(j), nStartTime, nTimeout); + found = true; + LogPrintf("Setting BIP9 activation parameters for %s to start=%ld, timeout=%ld\n", vDeploymentParams[0], nStartTime, nTimeout); + break; + } + } + if (!found) { + return InitError(strprintf("Invalid deployment (%s)", vDeploymentParams[0])); + } + } + } + // ********************************************************* Step 4: application initialization: dir lock, daemonize, pidfile, debug log // Initialize elliptic curve code @@ -1286,7 +1337,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) break; } - if (!fReindex) { + if (!fReindex && chainActive.Tip() != NULL) { uiInterface.InitMessage(_("Rewinding blocks...")); if (!RewindBlockIndex(chainparams)) { strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain"); @@ -1403,6 +1454,17 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) // ********************************************************* Step 10: import blocks + if (!CheckDiskSpace()) + return false; + + // Either install a handler to notify us when genesis activates, or set fHaveGenesis directly. + // No locking, as this happens before any background thread is started. + if (chainActive.Tip() == NULL) { + uiInterface.NotifyBlockTip.connect(BlockNotifyGenesisWait); + } else { + fHaveGenesis = true; + } + if (mapArgs.count("-blocknotify")) uiInterface.NotifyBlockTip.connect(BlockNotifyCallback); @@ -1412,26 +1474,20 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) BOOST_FOREACH(const std::string& strFile, mapMultiArgs["-loadblock"]) vImportFiles.push_back(strFile); } + threadGroup.create_thread(boost::bind(&ThreadImport, vImportFiles)); // Wait for genesis block to be processed - bool fHaveGenesis = false; - while (!fHaveGenesis && !fRequestShutdown) { - { - LOCK(cs_main); - fHaveGenesis = (chainActive.Tip() != NULL); - } - - if (!fHaveGenesis) { - MilliSleep(10); + { + boost::unique_lock<boost::mutex> lock(cs_GenesisWait); + while (!fHaveGenesis) { + condvar_GenesisWait.wait(lock); } + uiInterface.NotifyBlockTip.disconnect(BlockNotifyGenesisWait); } // ********************************************************* Step 11: start node - if (!CheckDiskSpace()) - return false; - if (!strErrors.str().empty()) return InitError(strErrors.str()); @@ -15,7 +15,7 @@ #include <vector> -/** +/** * secp256k1: * const unsigned int PRIVATE_KEY_SIZE = 279; * const unsigned int PUBLIC_KEY_SIZE = 65; @@ -45,6 +45,8 @@ private: //! The actual byte data unsigned char vch[32]; + static_assert(sizeof(vch) == 32, "vch must be 32 bytes in length to not break serialization"); + //! Check whether the 32-byte array pointed to be vch is valid keydata. bool static Check(const unsigned char* vch); @@ -70,20 +72,19 @@ public: friend bool operator==(const CKey& a, const CKey& b) { - return a.fCompressed == b.fCompressed && a.size() == b.size() && - memcmp(&a.vch[0], &b.vch[0], a.size()) == 0; + return a.fCompressed == b.fCompressed && + a.size() == b.size() && + memcmp(&a.vch[0], &b.vch[0], a.size()) == 0; } //! Initialize using begin and end iterators to byte data. template <typename T> void Set(const T pbegin, const T pend, bool fCompressedIn) { - if (pend - pbegin != 32) { + if (pend - pbegin != sizeof(vch)) { fValid = false; - return; - } - if (Check(&pbegin[0])) { - memcpy(vch, (unsigned char*)&pbegin[0], 32); + } else if (Check(&pbegin[0])) { + memcpy(vch, (unsigned char*)&pbegin[0], sizeof(vch)); fValid = true; fCompressed = fCompressedIn; } else { @@ -92,7 +93,7 @@ public: } //! Simple read-only vector-like interface. - unsigned int size() const { return (fValid ? 32 : 0); } + unsigned int size() const { return (fValid ? sizeof(vch) : 0); } const unsigned char* begin() const { return vch; } const unsigned char* end() const { return vch + size(); } @@ -110,7 +111,7 @@ public: /** * Convert the private key to a CPrivKey (serialized OpenSSL private key data). - * This is expensive. + * This is expensive. */ CPrivKey GetPrivKey() const; @@ -146,9 +147,6 @@ public: //! Load private key and check that public key matches. bool Load(CPrivKey& privkey, CPubKey& vchPubKey, bool fSkipCheck); - - //! Check whether an element of a signature (r or s) is valid. - static bool CheckSignatureElement(const unsigned char* vch, int len, bool half); }; struct CExtKey { @@ -160,8 +158,11 @@ struct CExtKey { friend bool operator==(const CExtKey& a, const CExtKey& b) { - return a.nDepth == b.nDepth && memcmp(&a.vchFingerprint[0], &b.vchFingerprint[0], 4) == 0 && a.nChild == b.nChild && - a.chaincode == b.chaincode && a.key == b.key; + return a.nDepth == b.nDepth && + memcmp(&a.vchFingerprint[0], &b.vchFingerprint[0], sizeof(vchFingerprint)) == 0 && + a.nChild == b.nChild && + a.chaincode == b.chaincode && + a.key == b.key; } void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const; diff --git a/src/main.cpp b/src/main.cpp index 73fbe53afb..3a07190a62 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -74,7 +74,6 @@ bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; bool fRequireStandard = true; -unsigned int nBytesPerSigOp = DEFAULT_BYTES_PER_SIGOP; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; size_t nCoinCacheUsage = 5000 * 300; @@ -107,11 +106,6 @@ map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(cs_main); map<COutPoint, set<map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main); void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** - * Returns true if there are nRequired or more blocks of minVersion or above - * in the last Consensus::Params::nMajorityWindow blocks, starting at pstart and going backwards. - */ -static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned nRequired, const Consensus::Params& consensusParams); static void CheckBlockIndex(const Consensus::Params& consensusParams); /** Constant stuff for coinbase transactions we create: */ @@ -694,8 +688,8 @@ bool AddOrphanTx(const CTransaction& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(c // have been mined or received. // 100 orphans, each of which is at most 99,999 bytes big is // at most 10 megabytes of orphans and somewhat more byprev index (in the worst case): - unsigned int sz = GetTransactionCost(tx); - if (sz >= MAX_STANDARD_TX_COST) + unsigned int sz = GetTransactionWeight(tx); + if (sz >= MAX_STANDARD_TX_WEIGHT) { LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); return false; @@ -788,7 +782,7 @@ bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) return true; if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) return true; - BOOST_FOREACH(const CTxIn& txin, tx.vin) { + for (const auto& txin : tx.vin) { if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) return false; } @@ -1002,11 +996,11 @@ bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool unsigned int GetLegacySigOpCount(const CTransaction& tx) { unsigned int nSigOps = 0; - BOOST_FOREACH(const CTxIn& txin, tx.vin) + for (const auto& txin : tx.vin) { nSigOps += txin.scriptSig.GetSigOpCount(false); } - BOOST_FOREACH(const CTxOut& txout, tx.vout) + for (const auto& txout : tx.vout) { nSigOps += txout.scriptPubKey.GetSigOpCount(false); } @@ -1064,7 +1058,7 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) // Check for negative or overflow output values CAmount nValueOut = 0; - BOOST_FOREACH(const CTxOut& txout, tx.vout) + for (const auto& txout : tx.vout) { if (txout.nValue < 0) return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); @@ -1077,7 +1071,7 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) // Check for duplicate inputs set<COutPoint> vInOutPoints; - BOOST_FOREACH(const CTxIn& txin, tx.vin) + for (const auto& txin : tx.vin) { if (vInOutPoints.count(txin.prevout)) return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); @@ -1091,7 +1085,7 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) } else { - BOOST_FOREACH(const CTxIn& txin, tx.vin) + for (const auto& txin : tx.vin) if (txin.prevout.IsNull()) return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); } @@ -1144,13 +1138,14 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C } // Reject transactions with witness before segregated witness activates (override with -prematurewitness) - if (!GetBoolArg("-prematurewitness",false) && !tx.wit.IsNull() && !IsWitnessEnabled(chainActive.Tip(), Params().GetConsensus())) { + bool witnessEnabled = IsWitnessEnabled(chainActive.Tip(), Params().GetConsensus()); + if (!GetBoolArg("-prematurewitness",false) && !tx.wit.IsNull() && !witnessEnabled) { return state.DoS(0, false, REJECT_NONSTANDARD, "no-witness-yet", true); } // Rather not work on nonstandard transactions (unless -testnet/-regtest) string reason; - if (fRequireStandard && !IsStandardTx(tx, reason)) + if (fRequireStandard && !IsStandardTx(tx, reason, witnessEnabled)) return state.DoS(0, false, REJECT_NONSTANDARD, reason); // Only accept nLockTime-using transactions that can be mined in the next @@ -1296,7 +1291,7 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const C // itself can contain sigops MAX_STANDARD_TX_SIGOPS is less than // MAX_BLOCK_SIGOPS; we still consider this an invalid rather than // merely non-standard transaction. - if ((nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) || (nBytesPerSigOp && nSigOpsCost > nSize * WITNESS_SCALE_FACTOR / nBytesPerSigOp)) + if (nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) return state.DoS(0, false, REJECT_NONSTANDARD, "bad-txns-too-many-sigops", false, strprintf("%d", nSigOpsCost)); @@ -2372,15 +2367,13 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin unsigned int flags = fStrictPayToScriptHash ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE; - // Start enforcing the DERSIG (BIP66) rules, for block.nVersion=3 blocks, - // when 75% of the network has upgraded: - if (block.nVersion >= 3 && IsSuperMajority(3, pindex->pprev, chainparams.GetConsensus().nMajorityEnforceBlockUpgrade, chainparams.GetConsensus())) { + // Start enforcing the DERSIG (BIP66) rule + if (pindex->nHeight >= chainparams.GetConsensus().BIP66Height) { flags |= SCRIPT_VERIFY_DERSIG; } - // Start enforcing CHECKLOCKTIMEVERIFY, (BIP65) for block.nVersion=4 - // blocks, when 75% of the network has upgraded: - if (block.nVersion >= 4 && IsSuperMajority(4, pindex->pprev, chainparams.GetConsensus().nMajorityEnforceBlockUpgrade, chainparams.GetConsensus())) { + // Start enforcing CHECKLOCKTIMEVERIFY (BIP65) rule + if (pindex->nHeight >= chainparams.GetConsensus().BIP65Height) { flags |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY; } @@ -3404,13 +3397,13 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P return state.DoS(100, false, REJECT_INVALID, "bad-cb-multiple", false, "more than one coinbase"); // Check transactions - BOOST_FOREACH(const CTransaction& tx, block.vtx) + for (const auto& tx : block.vtx) if (!CheckTransaction(tx, state)) return state.Invalid(false, state.GetRejectCode(), state.GetRejectReason(), strprintf("Transaction check failed (tx hash %s) %s", tx.GetHash().ToString(), state.GetDebugMessage())); unsigned int nSigOps = 0; - BOOST_FOREACH(const CTransaction& tx, block.vtx) + for (const auto& tx : block.vtx) { nSigOps += GetLegacySigOpCount(tx); } @@ -3502,8 +3495,9 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc return commitment; } -bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, CBlockIndex * const pindexPrev, int64_t nAdjustedTime) +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) { + const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; // Check proof of work if (block.nBits != GetNextWorkRequired(pindexPrev, &block, consensusParams)) return state.DoS(100, false, REJECT_INVALID, "bad-diffbits", false, "incorrect proof of work"); @@ -3517,18 +3511,19 @@ bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& sta return state.Invalid(false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); // Reject outdated version blocks when 95% (75% on testnet) of the network has upgraded: - for (int32_t version = 2; version < 5; ++version) // check for version 2, 3 and 4 upgrades - if (block.nVersion < version && IsSuperMajority(version, pindexPrev, consensusParams.nMajorityRejectBlockOutdated, consensusParams)) - return state.Invalid(false, REJECT_OBSOLETE, strprintf("bad-version(0x%08x)", version - 1), - strprintf("rejected nVersion=0x%08x block", version - 1)); + // check for version 2, 3 and 4 upgrades + if((block.nVersion < 2 && nHeight >= consensusParams.BIP34Height) || + (block.nVersion < 3 && nHeight >= consensusParams.BIP66Height) || + (block.nVersion < 4 && nHeight >= consensusParams.BIP65Height)) + return state.Invalid(false, REJECT_OBSOLETE, strprintf("bad-version(0x%08x)", block.nVersion), + strprintf("rejected nVersion=0x%08x block", block.nVersion)); return true; } -bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex * const pindexPrev) +bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; - const Consensus::Params& consensusParams = Params().GetConsensus(); // Start enforcing BIP113 (Median Time Past) using versionbits logic. int nLockTimeFlags = 0; @@ -3541,15 +3536,14 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn : block.GetBlockTime(); // Check that all transactions are finalized - BOOST_FOREACH(const CTransaction& tx, block.vtx) { + for (const auto& tx : block.vtx) { if (!IsFinalTx(tx, nHeight, nLockTimeCutoff)) { return state.DoS(10, false, REJECT_INVALID, "bad-txns-nonfinal", false, "non-final transaction"); } } - // Enforce block.nVersion=2 rule that the coinbase starts with serialized block height - // if 750 of the last 1,000 blocks are version 2 or greater (51/100 if testnet): - if (block.nVersion >= 2 && IsSuperMajority(2, pindexPrev, consensusParams.nMajorityEnforceBlockUpgrade, consensusParams)) + // Enforce rule that the coinbase starts with serialized block height + if (nHeight >= consensusParams.BIP34Height) { CScript expect = CScript() << nHeight; if (block.vtx[0].vin[0].scriptSig.size() < expect.size() || @@ -3567,7 +3561,7 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn // {0xaa, 0x21, 0xa9, 0xed}, and the following 32 bytes are SHA256^2(witness root, witness nonce). In case there are // multiple, the last one is used. bool fHaveWitness = false; - if (IsWitnessEnabled(pindexPrev, consensusParams)) { + if (VersionBitsState(pindexPrev, consensusParams, Consensus::DEPLOYMENT_SEGWIT, versionbitscache) == THRESHOLD_ACTIVE) { int commitpos = GetWitnessCommitmentIndex(block); if (commitpos != -1) { bool malleated = false; @@ -3576,11 +3570,11 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn // already does not permit it, it is impossible to trigger in the // witness tree. if (block.vtx[0].wit.vtxinwit.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0].size() != 32) { - return state.DoS(100, error("%s : invalid witness nonce size", __func__), REJECT_INVALID, "bad-witness-nonce-size", true); + return state.DoS(100, false, REJECT_INVALID, "bad-witness-nonce-size", true, strprintf("%s : invalid witness nonce size", __func__)); } CHash256().Write(hashWitness.begin(), 32).Write(&block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0][0], 32).Finalize(hashWitness.begin()); if (memcmp(hashWitness.begin(), &block.vtx[0].vout[commitpos].scriptPubKey[6], 32)) { - return state.DoS(100, error("%s : witness merkle commitment mismatch", __func__), REJECT_INVALID, "bad-witness-merkle-match", true); + return state.DoS(100, false, REJECT_INVALID, "bad-witness-merkle-match", true, strprintf("%s : witness merkle commitment mismatch", __func__)); } fHaveWitness = true; } @@ -3590,19 +3584,19 @@ bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIn if (!fHaveWitness) { for (size_t i = 0; i < block.vtx.size(); i++) { if (!block.vtx[i].wit.IsNull()) { - return state.DoS(100, error("%s : unexpected witness data found", __func__), REJECT_INVALID, "unexpected-witness", true); + return state.DoS(100, false, REJECT_INVALID, "unexpected-witness", true, strprintf("%s : unexpected witness data found", __func__)); } } } // After the coinbase witness nonce and commitment are verified, - // we can check if the block cost passes (before we've checked the - // coinbase witness, it would be possible for the cost to be too + // we can check if the block weight passes (before we've checked the + // coinbase witness, it would be possible for the weight to be too // large by filling up the coinbase witness, which doesn't change // the block hash, so we couldn't mark the block as permanently // failed). - if (GetBlockCost(block) > MAX_BLOCK_COST) { - return state.DoS(100, error("ContextualCheckBlock(): cost limit failed"), REJECT_INVALID, "bad-blk-cost"); + if (GetBlockWeight(block) > MAX_BLOCK_WEIGHT) { + return state.DoS(100, false, REJECT_INVALID, "bad-blk-weight", false, strprintf("%s : weight limit failed", __func__)); } return true; @@ -3689,7 +3683,8 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, const CCha } if (fNewBlock) *fNewBlock = true; - if ((!CheckBlock(block, state, chainparams.GetConsensus(), GetAdjustedTime())) || !ContextualCheckBlock(block, state, pindex->pprev)) { + if (!CheckBlock(block, state, chainparams.GetConsensus(), GetAdjustedTime()) || + !ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindex->pprev)) { if (state.IsInvalid() && !state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); @@ -3722,19 +3717,6 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, const CCha return true; } -static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned nRequired, const Consensus::Params& consensusParams) -{ - unsigned int nFound = 0; - for (int i = 0; i < consensusParams.nMajorityWindow && nFound < nRequired && pstart != NULL; i++) - { - if (pstart->nVersion >= minVersion) - ++nFound; - pstart = pstart->pprev; - } - return (nFound >= nRequired); -} - - bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp) { { @@ -3780,7 +3762,7 @@ bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, FormatStateMessage(state)); if (!CheckBlock(block, state, chainparams.GetConsensus(), fCheckPOW, fCheckMerkleRoot)) return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); - if (!ContextualCheckBlock(block, state, pindexPrev)) + if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) return false; @@ -4100,7 +4082,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, int nGoodTransactions = 0; CValidationState state; int reportDone = 0; - LogPrintf("[0%]..."); + LogPrintf("[0%%]..."); for (CBlockIndex* pindex = chainActive.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { boost::this_thread::interruption_point(); @@ -4331,8 +4313,6 @@ bool InitBlockIndex(const CChainParams& chainparams) CBlockIndex *pindex = AddToBlockIndex(block); if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) return error("LoadBlockIndex(): genesis block not accepted"); - if (!ActivateBestChain(state, chainparams, &block)) - return error("LoadBlockIndex(): genesis block cannot be activated"); // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); } catch (const std::runtime_error& e) { @@ -5343,7 +5323,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, BlockMap::iterator it = mapBlockIndex.find(req.blockhash); if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { - Misbehaving(pfrom->GetId(), 100); LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); return true; } @@ -5627,8 +5606,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); pfrom->PushMessage(NetMsgType::GETDATA, vInv); - return true; } + return true; } // If we're not close to tip yet, give up and let parallel block fetch work its magic @@ -6146,6 +6125,11 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } } + else if (strCommand == NetMsgType::NOTFOUND) { + // We do not care about the NOTFOUND message, but logging an Unknown Command + // message would be undesirable as we transmit it ourselves. + } + else { // Ignore unknown commands for extensibility LogPrint("net", "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); diff --git a/src/main.h b/src/main.h index 65ae2488f9..d4d70c0180 100644 --- a/src/main.h +++ b/src/main.h @@ -124,7 +124,6 @@ static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; /** Default for -permitbaremultisig */ static const bool DEFAULT_PERMIT_BAREMULTISIG = true; -static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; @@ -155,7 +154,7 @@ typedef boost::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern BlockMap mapBlockIndex; extern uint64_t nLastBlockTx; extern uint64_t nLastBlockSize; -extern uint64_t nLastBlockCost; +extern uint64_t nLastBlockWeight; extern const std::string strMessageMagic; extern CWaitableCriticalSection csBestBlock; extern CConditionVariable cvBlockChange; @@ -165,7 +164,6 @@ extern int nScriptCheckThreads; extern bool fTxIndex; extern bool fIsBareMultisigStd; extern bool fRequireStandard; -extern unsigned int nBytesPerSigOp; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; extern size_t nCoinCacheUsage; @@ -354,9 +352,22 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi /** Apply the effects of this transaction on the UTXO set represented by view */ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight); +/** Transaction validation functions */ + /** Context-independent validity checks */ bool CheckTransaction(const CTransaction& tx, CValidationState& state); +namespace Consensus { + +/** + * Check whether all inputs of this transaction are valid (no double spends and amounts) + * This does not modify the UTXO set. This does not check scripts and sigs. + * Preconditions: tx.IsCoinBase() is false. + */ +bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight); + +} // namespace Consensus + /** * Check if transaction is final and can be included in a block with the * specified height and time. Consensus critical. @@ -447,8 +458,8 @@ bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::P /** Context-dependent validity checks. * By "context", we mean only the previous block headers, but not the UTXO * set; UTXO-related validity checks are done in ConnectBlock(). */ -bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, CBlockIndex* pindexPrev, int64_t nAdjustedTime); -bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex *pindexPrev); +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev, int64_t nAdjustedTime); +bool ContextualCheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); /** Apply the effects of this block (with given index) on the UTXO set represented by coins. * Validity checks that depend on the UTXO set are also done; ConnectBlock() diff --git a/src/miner.cpp b/src/miner.cpp index 8153fb9f9e..9575858840 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -45,7 +45,7 @@ using namespace std; uint64_t nLastBlockTx = 0; uint64_t nLastBlockSize = 0; -uint64_t nLastBlockCost = 0; +uint64_t nLastBlockWeight = 0; class ScoreCompare { @@ -77,30 +77,30 @@ BlockAssembler::BlockAssembler(const CChainParams& _chainparams) : chainparams(_chainparams) { // Block resource limits - // If neither -blockmaxsize or -blockmaxcost is given, limit to DEFAULT_BLOCK_MAX_* + // If neither -blockmaxsize or -blockmaxweight is given, limit to DEFAULT_BLOCK_MAX_* // If only one is given, only restrict the specified resource. // If both are given, restrict both. - nBlockMaxCost = DEFAULT_BLOCK_MAX_COST; + nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; nBlockMaxSize = DEFAULT_BLOCK_MAX_SIZE; - bool fCostSet = false; - if (mapArgs.count("-blockmaxcost")) { - nBlockMaxCost = GetArg("-blockmaxcost", DEFAULT_BLOCK_MAX_COST); + bool fWeightSet = false; + if (mapArgs.count("-blockmaxweight")) { + nBlockMaxWeight = GetArg("-blockmaxweight", DEFAULT_BLOCK_MAX_WEIGHT); nBlockMaxSize = MAX_BLOCK_SERIALIZED_SIZE; - fCostSet = true; + fWeightSet = true; } if (mapArgs.count("-blockmaxsize")) { nBlockMaxSize = GetArg("-blockmaxsize", DEFAULT_BLOCK_MAX_SIZE); - if (!fCostSet) { - nBlockMaxCost = nBlockMaxSize * WITNESS_SCALE_FACTOR; + if (!fWeightSet) { + nBlockMaxWeight = nBlockMaxSize * WITNESS_SCALE_FACTOR; } } - // Limit cost to between 4K and MAX_BLOCK_COST-4K for sanity: - nBlockMaxCost = std::max((unsigned int)4000, std::min((unsigned int)(MAX_BLOCK_COST-4000), nBlockMaxCost)); + // Limit weight to between 4K and MAX_BLOCK_WEIGHT-4K for sanity: + nBlockMaxWeight = std::max((unsigned int)4000, std::min((unsigned int)(MAX_BLOCK_WEIGHT-4000), nBlockMaxWeight)); // Limit size to between 1K and MAX_BLOCK_SERIALIZED_SIZE-1K for sanity: nBlockMaxSize = std::max((unsigned int)1000, std::min((unsigned int)(MAX_BLOCK_SERIALIZED_SIZE-1000), nBlockMaxSize)); - // Whether we need to account for byte usage (in addition to cost usage) + // Whether we need to account for byte usage (in addition to weight usage) fNeedSizeAccounting = (nBlockMaxSize < MAX_BLOCK_SERIALIZED_SIZE-1000); } @@ -110,7 +110,7 @@ void BlockAssembler::resetBlock() // Reserve space for coinbase tx nBlockSize = 1000; - nBlockCost = 4000; + nBlockWeight = 4000; nBlockSigOpsCost = 400; fIncludeWitness = false; @@ -167,7 +167,7 @@ CBlockTemplate* BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn) nLastBlockTx = nBlockTx; nLastBlockSize = nBlockSize; - nLastBlockCost = nBlockCost; + nLastBlockWeight = nBlockWeight; LogPrintf("CreateNewBlock(): total size %u txs: %u fees: %ld sigops %d\n", nBlockSize, nBlockTx, nFees, nBlockSigOpsCost); // Create coinbase transaction. @@ -187,7 +187,7 @@ CBlockTemplate* BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn) UpdateTime(pblock, chainparams.GetConsensus(), pindexPrev); pblock->nBits = GetNextWorkRequired(pindexPrev, pblock, chainparams.GetConsensus()); pblock->nNonce = 0; - pblocktemplate->vTxSigOpsCost[0] = GetLegacySigOpCount(pblock->vtx[0]); + pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(pblock->vtx[0]); CValidationState state; if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { @@ -223,8 +223,8 @@ void BlockAssembler::onlyUnconfirmed(CTxMemPool::setEntries& testSet) bool BlockAssembler::TestPackage(uint64_t packageSize, int64_t packageSigOpsCost) { - // TODO: switch to cost-based accounting for packages instead of vsize-based accounting. - if (nBlockCost + WITNESS_SCALE_FACTOR * packageSize >= nBlockMaxCost) + // TODO: switch to weight-based accounting for packages instead of vsize-based accounting. + if (nBlockWeight + WITNESS_SCALE_FACTOR * packageSize >= nBlockMaxWeight) return false; if (nBlockSigOpsCost + packageSigOpsCost >= MAX_BLOCK_SIGOPS_COST) return false; @@ -257,17 +257,17 @@ bool BlockAssembler::TestPackageTransactions(const CTxMemPool::setEntries& packa bool BlockAssembler::TestForBlock(CTxMemPool::txiter iter) { - if (nBlockCost + iter->GetTxCost() >= nBlockMaxCost) { + if (nBlockWeight + iter->GetTxWeight() >= nBlockMaxWeight) { // If the block is so close to full that no more txs will fit // or if we've tried more than 50 times to fill remaining space // then flag that the block is finished - if (nBlockCost > nBlockMaxCost - 400 || lastFewTxs > 50) { + if (nBlockWeight > nBlockMaxWeight - 400 || lastFewTxs > 50) { blockFinished = true; return false; } - // Once we're within 4000 cost of a full block, only look at 50 more txs + // Once we're within 4000 weight of a full block, only look at 50 more txs // to try to fill the remaining space. - if (nBlockCost > nBlockMaxCost - 4000) { + if (nBlockWeight > nBlockMaxWeight - 4000) { lastFewTxs++; } return false; @@ -315,7 +315,7 @@ void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) if (fNeedSizeAccounting) { nBlockSize += ::GetSerializeSize(iter->GetTx(), SER_NETWORK, PROTOCOL_VERSION); } - nBlockCost += iter->GetTxCost(); + nBlockWeight += iter->GetTxWeight(); ++nBlockTx; nBlockSigOpsCost += iter->GetSigOpCost(); nFees += iter->GetFee(); diff --git a/src/miner.h b/src/miner.h index d16e37bb59..11753f5e43 100644 --- a/src/miner.h +++ b/src/miner.h @@ -141,11 +141,11 @@ private: // Configuration parameters for the block size bool fIncludeWitness; - unsigned int nBlockMaxCost, nBlockMaxSize; + unsigned int nBlockMaxWeight, nBlockMaxSize; bool fNeedSizeAccounting; // Information on the current status of the block - uint64_t nBlockCost; + uint64_t nBlockWeight; uint64_t nBlockSize; uint64_t nBlockTx; uint64_t nBlockSigOpsCost; diff --git a/src/net.cpp b/src/net.cpp index 4cbc43e4d8..4bbe5059df 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -982,11 +982,11 @@ static bool AttemptToEvictConnection() { uint64_t naMostConnections; unsigned int nMostConnections = 0; int64_t nMostConnectionsTime = 0; - std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapAddrCounts; + std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapNetGroupNodes; BOOST_FOREACH(const NodeEvictionCandidate &node, vEvictionCandidates) { - mapAddrCounts[node.nKeyedNetGroup].push_back(node); - int64_t grouptime = mapAddrCounts[node.nKeyedNetGroup][0].nTimeConnected; - size_t groupsize = mapAddrCounts[node.nKeyedNetGroup].size(); + mapNetGroupNodes[node.nKeyedNetGroup].push_back(node); + int64_t grouptime = mapNetGroupNodes[node.nKeyedNetGroup][0].nTimeConnected; + size_t groupsize = mapNetGroupNodes[node.nKeyedNetGroup].size(); if (groupsize > nMostConnections || (groupsize == nMostConnections && grouptime > nMostConnectionsTime)) { nMostConnections = groupsize; @@ -996,7 +996,7 @@ static bool AttemptToEvictConnection() { } // Reduce to the network group with the most connections - vEvictionCandidates = std::move(mapAddrCounts[naMostConnections]); + vEvictionCandidates = std::move(mapNetGroupNodes[naMostConnections]); // Disconnect from the network group with the most connections NodeId evicted = vEvictionCandidates.front().id; @@ -2050,6 +2050,8 @@ void StartNode(boost::thread_group& threadGroup, CScheduler& scheduler) DumpBanlist(); } + uiInterface.InitMessage(_("Starting network threads...")); + fAddressesInitialized = true; if (semOutbound == NULL) { diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index f2148bfe10..48080abc77 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -31,7 +31,7 @@ * DUP CHECKSIG DROP ... repeated 100 times... OP_1 */ -bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) +bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType, const bool witnessEnabled) { std::vector<std::vector<unsigned char> > vSolutions; if (!Solver(scriptPubKey, whichType, vSolutions)) @@ -50,10 +50,13 @@ bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) (!fAcceptDatacarrier || scriptPubKey.size() > nMaxDatacarrierBytes)) return false; + else if (!witnessEnabled && (whichType == TX_WITNESS_V0_KEYHASH || whichType == TX_WITNESS_V0_SCRIPTHASH)) + return false; + return whichType != TX_NONSTANDARD; } -bool IsStandardTx(const CTransaction& tx, std::string& reason) +bool IsStandardTx(const CTransaction& tx, std::string& reason, const bool witnessEnabled) { if (tx.nVersion > CTransaction::MAX_STANDARD_VERSION || tx.nVersion < 1) { reason = "version"; @@ -64,8 +67,8 @@ bool IsStandardTx(const CTransaction& tx, std::string& reason) // almost as much to process as they cost the sender in fees, because // computing signature hashes is O(ninputs*txsize). Limiting transactions // to MAX_STANDARD_TX_SIZE mitigates CPU exhaustion attacks. - unsigned int sz = GetTransactionCost(tx); - if (sz >= MAX_STANDARD_TX_COST) { + unsigned int sz = GetTransactionWeight(tx); + if (sz >= MAX_STANDARD_TX_WEIGHT) { reason = "tx-size"; return false; } @@ -92,7 +95,7 @@ bool IsStandardTx(const CTransaction& tx, std::string& reason) unsigned int nDataOut = 0; txnouttype whichType; BOOST_FOREACH(const CTxOut& txout, tx.vout) { - if (!::IsStandard(txout.scriptPubKey, whichType)) { + if (!::IsStandard(txout.scriptPubKey, whichType, witnessEnabled)) { reason = "scriptpubkey"; return false; } @@ -151,12 +154,14 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) return true; } -int64_t GetVirtualTransactionSize(int64_t nCost) +unsigned int nBytesPerSigOp = DEFAULT_BYTES_PER_SIGOP; + +int64_t GetVirtualTransactionSize(int64_t nWeight, int64_t nSigOpCost) { - return (nCost + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR; + return (std::max(nWeight, nSigOpCost * nBytesPerSigOp) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR; } -int64_t GetVirtualTransactionSize(const CTransaction& tx) +int64_t GetVirtualTransactionSize(const CTransaction& tx, int64_t nSigOpCost) { - return GetVirtualTransactionSize(GetTransactionCost(tx)); + return GetVirtualTransactionSize(GetTransactionWeight(tx), nSigOpCost); } diff --git a/src/policy/policy.h b/src/policy/policy.h index 29a8cc57c2..6bf5ca0ee5 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -18,16 +18,18 @@ class CCoinsViewCache; static const unsigned int DEFAULT_BLOCK_MAX_SIZE = 750000; /** Default for -blockprioritysize, maximum space for zero/low-fee transactions **/ static const unsigned int DEFAULT_BLOCK_PRIORITY_SIZE = 0; -/** Default for -blockmaxcost, which control the range of block costs the mining code will create **/ -static const unsigned int DEFAULT_BLOCK_MAX_COST = 3000000; -/** The maximum size for transactions we're willing to relay/mine */ -static const unsigned int MAX_STANDARD_TX_COST = 400000; +/** Default for -blockmaxweight, which controls the range of block weights the mining code will create **/ +static const unsigned int DEFAULT_BLOCK_MAX_WEIGHT = 3000000; +/** The maximum weight for transactions we're willing to relay/mine */ +static const unsigned int MAX_STANDARD_TX_WEIGHT = 400000; /** Maximum number of signature check operations in an IsStandard() P2SH script */ static const unsigned int MAX_P2SH_SIGOPS = 15; /** The maximum number of sigops we're willing to relay/mine in a single tx */ static const unsigned int MAX_STANDARD_TX_SIGOPS_COST = MAX_BLOCK_SIGOPS_COST/5; /** Default for -maxmempool, maximum megabytes of mempool memory usage */ static const unsigned int DEFAULT_MAX_MEMPOOL_SIZE = 300; +/** Default for -bytespersigop */ +static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; /** * Standard script verification flags that standard transactions will comply * with. However scripts violating these flags may still be present in valid @@ -53,12 +55,12 @@ static const unsigned int STANDARD_NOT_MANDATORY_VERIFY_FLAGS = STANDARD_SCRIPT_ static const unsigned int STANDARD_LOCKTIME_VERIFY_FLAGS = LOCKTIME_VERIFY_SEQUENCE | LOCKTIME_MEDIAN_TIME_PAST; -bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType); +bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType, const bool witnessEnabled = false); /** * Check for standard transaction types * @return True if all outputs (scriptPubKeys) use only standard transaction forms */ -bool IsStandardTx(const CTransaction& tx, std::string& reason); +bool IsStandardTx(const CTransaction& tx, std::string& reason, const bool witnessEnabled = false); /** * Check for standard transaction types * @param[in] mapInputs Map of previous transactions that have outputs we're spending @@ -66,8 +68,10 @@ bool IsStandardTx(const CTransaction& tx, std::string& reason); */ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs); -/** Compute the virtual transaction size (cost reinterpreted as bytes). */ -int64_t GetVirtualTransactionSize(int64_t nCost); -int64_t GetVirtualTransactionSize(const CTransaction& tx); +extern unsigned int nBytesPerSigOp; + +/** Compute the virtual transaction size (weight reinterpreted as bytes). */ +int64_t GetVirtualTransactionSize(int64_t nWeight, int64_t nSigOpCost); +int64_t GetVirtualTransactionSize(const CTransaction& tx, int64_t nSigOpCost = 0); #endif // BITCOIN_POLICY_POLICY_H diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index df900388f2..0e6ab4dd71 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -32,11 +32,11 @@ std::string CBlock::ToString() const return s.str(); } -int64_t GetBlockCost(const CBlock& block) +int64_t GetBlockWeight(const CBlock& block) { - // This implements the cost = (stripped_size * 4) + witness_size formula, + // This implements the weight = (stripped_size * 4) + witness_size formula, // using only serialization with and without witness data. As witness_size // is equal to total_size - stripped_size, this formula is identical to: - // cost = (stripped_size * 3) + total_size. + // weight = (stripped_size * 3) + total_size. return ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION); } diff --git a/src/primitives/block.h b/src/primitives/block.h index e2a309e63d..72dfed985a 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -154,7 +154,7 @@ struct CBlockLocator } }; -/** Compute the consensus-critical block cost (see BIP 141). */ -int64_t GetBlockCost(const CBlock& tx); +/** Compute the consensus-critical block weight (see BIP 141). */ +int64_t GetBlockWeight(const CBlock& tx); #endif // BITCOIN_PRIMITIVES_BLOCK_H diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 7f10409c05..8d63805643 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -121,7 +121,7 @@ unsigned int CTransaction::CalculateModifiedSize(unsigned int nTxSize) const // Providing any more cleanup incentive than making additional inputs free would // risk encouraging people to create junk outputs to redeem later. if (nTxSize == 0) - nTxSize = (GetTransactionCost(*this) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR; + nTxSize = (GetTransactionWeight(*this) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR; for (std::vector<CTxIn>::const_iterator it(vin.begin()); it != vin.end(); ++it) { unsigned int offset = 41U + std::min(110U, (unsigned int)it->scriptSig.size()); @@ -149,7 +149,7 @@ std::string CTransaction::ToString() const return str; } -int64_t GetTransactionCost(const CTransaction& tx) +int64_t GetTransactionWeight(const CTransaction& tx) { return ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR -1) + ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION); } diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 8a2d5dd22c..5689d15bf7 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -290,6 +290,8 @@ struct CMutableTransaction; */ template<typename Stream, typename Operation, typename TxType> inline void SerializeTransaction(TxType& tx, Stream& s, Operation ser_action, int nType, int nVersion) { + const bool fAllowWitness = !(nVersion & SERIALIZE_TRANSACTION_NO_WITNESS); + READWRITE(*const_cast<int32_t*>(&tx.nVersion)); unsigned char flags = 0; if (ser_action.ForRead()) { @@ -298,7 +300,7 @@ inline void SerializeTransaction(TxType& tx, Stream& s, Operation ser_action, in const_cast<CTxWitness*>(&tx.wit)->SetNull(); /* Try to read the vin. In case the dummy is there, this will be read as an empty vector. */ READWRITE(*const_cast<std::vector<CTxIn>*>(&tx.vin)); - if (tx.vin.size() == 0 && !(nVersion & SERIALIZE_TRANSACTION_NO_WITNESS)) { + if (tx.vin.size() == 0 && fAllowWitness) { /* We read a dummy or an empty vin. */ READWRITE(flags); if (flags != 0) { @@ -309,7 +311,7 @@ inline void SerializeTransaction(TxType& tx, Stream& s, Operation ser_action, in /* We read a non-empty vin. Assume a normal vout follows. */ READWRITE(*const_cast<std::vector<CTxOut>*>(&tx.vout)); } - if ((flags & 1) && !(nVersion & SERIALIZE_TRANSACTION_NO_WITNESS)) { + if ((flags & 1) && fAllowWitness) { /* The witness flag is present, and we support witnesses. */ flags ^= 1; const_cast<CTxWitness*>(&tx.wit)->vtxinwit.resize(tx.vin.size()); @@ -322,7 +324,7 @@ inline void SerializeTransaction(TxType& tx, Stream& s, Operation ser_action, in } else { // Consistency check assert(tx.wit.vtxinwit.size() <= tx.vin.size()); - if (!(nVersion & SERIALIZE_TRANSACTION_NO_WITNESS)) { + if (fAllowWitness) { /* Check whether witnesses need to be serialized. */ if (!tx.wit.IsNull()) { flags |= 1; @@ -459,7 +461,7 @@ struct CMutableTransaction uint256 GetHash() const; }; -/** Compute the cost of a transaction, as defined by BIP 141 */ -int64_t GetTransactionCost(const CTransaction &tx); +/** Compute the weight of a transaction, as defined by BIP 141 */ +int64_t GetTransactionWeight(const CTransaction &tx); #endif // BITCOIN_PRIMITIVES_TRANSACTION_H diff --git a/src/pubkey.h b/src/pubkey.h index db5444ea9d..aebfdbc826 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -13,7 +13,7 @@ #include <stdexcept> #include <vector> -/** +/** * secp256k1: * const unsigned int PRIVATE_KEY_SIZE = 279; * const unsigned int PUBLIC_KEY_SIZE = 65; @@ -156,7 +156,7 @@ public: /* * Check syntactic correctness. - * + * * Note that this is consensus critical as CheckSig() calls it! */ bool IsValid() const @@ -203,8 +203,11 @@ struct CExtPubKey { friend bool operator==(const CExtPubKey &a, const CExtPubKey &b) { - return a.nDepth == b.nDepth && memcmp(&a.vchFingerprint[0], &b.vchFingerprint[0], 4) == 0 && a.nChild == b.nChild && - a.chaincode == b.chaincode && a.pubkey == b.pubkey; + return a.nDepth == b.nDepth && + memcmp(&a.vchFingerprint[0], &b.vchFingerprint[0], sizeof(vchFingerprint)) == 0 && + a.nChild == b.nChild && + a.chaincode == b.chaincode && + a.pubkey == b.pubkey; } void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const; diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 9042e3b56a..2afefb733e 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -902,17 +902,22 @@ void BitcoinGUI::closeEvent(QCloseEvent *event) #ifndef Q_OS_MAC // Ignored on Mac if(clientModel && clientModel->getOptionsModel()) { - if(!clientModel->getOptionsModel()->getMinimizeToTray() && - !clientModel->getOptionsModel()->getMinimizeOnClose()) + if(!clientModel->getOptionsModel()->getMinimizeOnClose()) { // close rpcConsole in case it was open to make some space for the shutdown window rpcConsole->close(); QApplication::quit(); } + else + { + QMainWindow::showMinimized(); + event->ignore(); + } } -#endif +#else QMainWindow::closeEvent(event); +#endif } void BitcoinGUI::showEvent(QShowEvent *event) diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index cc2cbc0e66..684db71a8c 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -43,6 +43,8 @@ void OptionsModel::Init(bool resetSettings) if (resetSettings) Reset(); + checkAndMigrate(); + QSettings settings; // Ensure restart flag is unset on client startup @@ -429,3 +431,22 @@ bool OptionsModel::isRestartRequired() QSettings settings; return settings.value("fRestartRequired", false).toBool(); } + +void OptionsModel::checkAndMigrate() +{ + // Migration of default values + // Check if the QSettings container was already loaded with this client version + QSettings settings; + static const char strSettingsVersionKey[] = "nSettingsVersion"; + int settingsVersion = settings.contains(strSettingsVersionKey) ? settings.value(strSettingsVersionKey).toInt() : 0; + if (settingsVersion < CLIENT_VERSION) + { + // -dbcache was bumped from 100 to 300 in 0.13 + // see https://github.com/bitcoin/bitcoin/pull/8273 + // force people to upgrade to the new value if they are using 100MB + if (settingsVersion < 130000 && settings.contains("nDatabaseCache") && settings.value("nDatabaseCache").toLongLong() == 100) + settings.setValue("nDatabaseCache", (qint64)nDefaultDbCache); + + settings.setValue(strSettingsVersionKey, CLIENT_VERSION); + } +}
\ No newline at end of file diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 3b491ceac2..b23b5f2607 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -84,9 +84,11 @@ private: /* settings that were overriden by command-line */ QString strOverriddenByCommandLine; - /// Add option to list of GUI options overridden through command line/config file + // Add option to list of GUI options overridden through command line/config file void addOverriddenOption(const std::string &option); + // Check settings version and upgrade default values if required + void checkAndMigrate(); Q_SIGNALS: void displayUnitChanged(int unit); void coinControlFeaturesChanged(bool); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 20eefa1c57..e3c32d905a 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -101,7 +101,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool tx result.push_back(Pair("confirmations", confirmations)); result.push_back(Pair("strippedsize", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS))); result.push_back(Pair("size", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION))); - result.push_back(Pair("cost", (int)::GetBlockCost(block))); + result.push_back(Pair("weight", (int)::GetBlockWeight(block))); result.push_back(Pair("height", blockindex->nHeight)); result.push_back(Pair("version", block.nVersion)); result.push_back(Pair("versionHex", strprintf("%08x", block.nVersion))); @@ -559,7 +559,7 @@ UniValue getblock(const UniValue& params, bool fHelp) " \"confirmations\" : n, (numeric) The number of confirmations, or -1 if the block is not on the main chain\n" " \"size\" : n, (numeric) The block size\n" " \"strippedsize\" : n, (numeric) The block size excluding witness data\n" - " \"cost\" : n (numeric) The block cost\n" + " \"weight\" : n (numeric) The block weight (BIP 141)\n" " \"height\" : n, (numeric) The block height or index\n" " \"version\" : n, (numeric) The block version\n" " \"versionHex\" : \"00000000\", (string) The block version formatted in hexadecimal\n" @@ -817,22 +817,23 @@ UniValue verifychain(const UniValue& params, bool fHelp) } /** Implementation of IsSuperMajority with better feedback */ -static UniValue SoftForkMajorityDesc(int minVersion, CBlockIndex* pindex, int nRequired, const Consensus::Params& consensusParams) +static UniValue SoftForkMajorityDesc(int version, CBlockIndex* pindex, const Consensus::Params& consensusParams) { - int nFound = 0; - CBlockIndex* pstart = pindex; - for (int i = 0; i < consensusParams.nMajorityWindow && pstart != NULL; i++) + UniValue rv(UniValue::VOBJ); + bool activated = false; + switch(version) { - if (pstart->nVersion >= minVersion) - ++nFound; - pstart = pstart->pprev; + case 2: + activated = pindex->nHeight >= consensusParams.BIP34Height; + break; + case 3: + activated = pindex->nHeight >= consensusParams.BIP66Height; + break; + case 4: + activated = pindex->nHeight >= consensusParams.BIP65Height; + break; } - - UniValue rv(UniValue::VOBJ); - rv.push_back(Pair("status", nFound >= nRequired)); - rv.push_back(Pair("found", nFound)); - rv.push_back(Pair("required", nRequired)); - rv.push_back(Pair("window", consensusParams.nMajorityWindow)); + rv.push_back(Pair("status", activated)); return rv; } @@ -841,8 +842,7 @@ static UniValue SoftForkDesc(const std::string &name, int version, CBlockIndex* UniValue rv(UniValue::VOBJ); rv.push_back(Pair("id", name)); rv.push_back(Pair("version", version)); - rv.push_back(Pair("enforce", SoftForkMajorityDesc(version, pindex, consensusParams.nMajorityEnforceBlockUpgrade, consensusParams))); - rv.push_back(Pair("reject", SoftForkMajorityDesc(version, pindex, consensusParams.nMajorityRejectBlockOutdated, consensusParams))); + rv.push_back(Pair("reject", SoftForkMajorityDesc(version, pindex, consensusParams))); return rv; } @@ -897,13 +897,9 @@ UniValue getblockchaininfo(const UniValue& params, bool fHelp) " {\n" " \"id\": \"xxxx\", (string) name of softfork\n" " \"version\": xx, (numeric) block version\n" - " \"enforce\": { (object) progress toward enforcing the softfork rules for new-version blocks\n" + " \"reject\": { (object) progress toward rejecting pre-softfork blocks\n" " \"status\": xx, (boolean) true if threshold reached\n" - " \"found\": xx, (numeric) number of blocks with the new version found\n" - " \"required\": xx, (numeric) number of blocks required to trigger\n" - " \"window\": xx, (numeric) maximum size of examined window of recent blocks\n" " },\n" - " \"reject\": { ... } (object) progress toward rejecting pre-softfork blocks (same fields as \"enforce\")\n" " }, ...\n" " ],\n" " \"bip9_softforks\": { (object) status of BIP9 softforks in progress\n" diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 4c4e599781..2479e5d595 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -224,7 +224,7 @@ UniValue getmininginfo(const UniValue& params, bool fHelp) "{\n" " \"blocks\": nnn, (numeric) The current block\n" " \"currentblocksize\": nnn, (numeric) The last block size\n" - " \"currentblockcost\": nnn, (numeric) The last block cost\n" + " \"currentblockweight\": nnn, (numeric) The last block weight\n" " \"currentblocktx\": nnn, (numeric) The last block transaction\n" " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" " \"errors\": \"...\" (string) Current errors\n" @@ -243,7 +243,7 @@ UniValue getmininginfo(const UniValue& params, bool fHelp) UniValue obj(UniValue::VOBJ); obj.push_back(Pair("blocks", (int)chainActive.Height())); obj.push_back(Pair("currentblocksize", (uint64_t)nLastBlockSize)); - obj.push_back(Pair("currentblockcost", (uint64_t)nLastBlockCost)); + obj.push_back(Pair("currentblockweight", (uint64_t)nLastBlockWeight)); obj.push_back(Pair("currentblocktx", (uint64_t)nLastBlockTx)); obj.push_back(Pair("difficulty", (double)GetDifficulty())); obj.push_back(Pair("errors", GetWarnings("statusbar"))); @@ -358,7 +358,7 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) " ],\n" " \"fee\": n, (numeric) difference in value between transaction inputs and outputs (in Satoshis); for coinbase transactions, this is a negative Number of the total collected block fees (ie, not including the block subsidy); if key is not present, fee is unknown and clients MUST NOT assume there isn't one\n" " \"sigops\" : n, (numeric) total SigOps cost, as counted for purposes of block limits; if key is not present, sigop cost is unknown and clients MUST NOT assume it is zero\n" - " \"cost\" : n, (numeric) total transaction size cost, as counted for purposes of block limits\n" + " \"weight\" : n, (numeric) total transaction weight, as counted for purposes of block limits\n" " \"required\" : true|false (boolean) if provided and true, this transaction must be in the final block\n" " }\n" " ,...\n" @@ -377,7 +377,7 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) " \"noncerange\" : \"00000000ffffffff\", (string) A range of valid nonces\n" " \"sigoplimit\" : n, (numeric) cost limit of sigops in blocks\n" " \"sizelimit\" : n, (numeric) limit of block size\n" - " \"costlimit\" : n, (numeric) limit of block cost\n" + " \"weightlimit\" : n, (numeric) limit of block weight\n" " \"curtime\" : ttt, (numeric) current timestamp in seconds since epoch (Jan 1 1970 GMT)\n" " \"bits\" : \"xxx\", (string) compressed target of next block\n" " \"height\" : n (numeric) The height of the next block\n" @@ -546,6 +546,9 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) UpdateTime(pblock, consensusParams, pindexPrev); pblock->nNonce = 0; + // NOTE: If at some point we support pre-segwit miners post-segwit-activation, this needs to take segwit support into consideration + const bool fPreSegWit = (THRESHOLD_ACTIVE != VersionBitsState(pindexPrev, consensusParams, Consensus::DEPLOYMENT_SEGWIT, versionbitscache)); + UniValue aCaps(UniValue::VARR); aCaps.push_back("proposal"); UniValue transactions(UniValue::VARR); @@ -574,8 +577,13 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) int index_in_template = i - 1; entry.push_back(Pair("fee", pblocktemplate->vTxFees[index_in_template])); - entry.push_back(Pair("sigops", pblocktemplate->vTxSigOpsCost[index_in_template])); - entry.push_back(Pair("cost", GetTransactionCost(tx))); + int64_t nTxSigOps = pblocktemplate->vTxSigOpsCost[index_in_template]; + if (fPreSegWit) { + assert(nTxSigOps % WITNESS_SCALE_FACTOR == 0); + nTxSigOps /= WITNESS_SCALE_FACTOR; + } + entry.push_back(Pair("sigops", nTxSigOps)); + entry.push_back(Pair("weight", GetTransactionWeight(tx))); transactions.push_back(entry); } @@ -657,9 +665,14 @@ UniValue getblocktemplate(const UniValue& params, bool fHelp) result.push_back(Pair("mintime", (int64_t)pindexPrev->GetMedianTimePast()+1)); result.push_back(Pair("mutable", aMutable)); result.push_back(Pair("noncerange", "00000000ffffffff")); - result.push_back(Pair("sigoplimit", (int64_t)MAX_BLOCK_SIGOPS_COST)); + int64_t nSigOpLimit = MAX_BLOCK_SIGOPS_COST; + if (fPreSegWit) { + assert(nSigOpLimit % WITNESS_SCALE_FACTOR == 0); + nSigOpLimit /= WITNESS_SCALE_FACTOR; + } + result.push_back(Pair("sigoplimit", nSigOpLimit)); result.push_back(Pair("sizelimit", (int64_t)MAX_BLOCK_SERIALIZED_SIZE)); - result.push_back(Pair("costlimit", (int64_t)MAX_BLOCK_COST)); + result.push_back(Pair("weightlimit", (int64_t)MAX_BLOCK_WEIGHT)); result.push_back(Pair("curtime", pblock->GetBlockTime())); result.push_back(Pair("bits", strprintf("%08x", pblock->nBits))); result.push_back(Pair("height", (int64_t)(pindexPrev->nHeight+1))); diff --git a/src/script/bitcoinconsensus.h b/src/script/bitcoinconsensus.h index 6f868d0d6d..f73a8e30bc 100644 --- a/src/script/bitcoinconsensus.h +++ b/src/script/bitcoinconsensus.h @@ -51,6 +51,7 @@ enum bitcoinconsensus_SCRIPT_FLAGS_VERIFY_P2SH = (1U << 0), // evaluate P2SH (BIP16) subscripts bitcoinconsensus_SCRIPT_FLAGS_VERIFY_DERSIG = (1U << 2), // enforce strict DER (BIP66) compliance bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKLOCKTIMEVERIFY = (1U << 9), // enable CHECKLOCKTIMEVERIFY (BIP65) + bitcoinconsensus_SCRIPT_FLAGS_VERIFY_CHECKSEQUENCEVERIFY = (1U << 10), // enable CHECKSEQUENCEVERIFY (BIP112) bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS = (1U << 11), // enable WITNESS (BIP141) }; diff --git a/src/test/README.md b/src/test/README.md index b2d6be14f1..61462642bf 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -5,18 +5,15 @@ sense to simply use this framework rather than require developers to configure some other framework (we want as few impediments to creating unit tests as possible). -The build system is setup to compile an executable called "test_bitcoin" +The build system is setup to compile an executable called `test_bitcoin` that runs all of the unit tests. The main source file is called -test_bitcoin.cpp, which simply includes other files that contain the -actual unit tests (outside of a couple required preprocessor -directives). The pattern is to create one test file for each class or -source file for which you want to create unit tests. The file naming -convention is "<source_filename>_tests.cpp" and such files should wrap -their tests in a test suite called "<source_filename>_tests". For an -examples of this pattern, examine uint160_tests.cpp and -uint256_tests.cpp. - -Add the source files to /src/Makefile.test.include to add them to the build. +test_bitcoin.cpp. To add a new unit test file to our test suite you need +to add the file to `src/Makefile.test.include`. The pattern is to create +one test file for each class or source file for which you want to create +unit tests. The file naming convention is `<source_filename>_tests.cpp` +and such files should wrap their tests in a test suite +called `<source_filename>_tests`. For an example of this pattern, +examine `uint256_tests.cpp`. For further reading, I found the following website to be helpful in explaining how the boost unit test framework works: @@ -31,5 +28,5 @@ example, to run just the getarg_tests verbosely: test_bitcoin --run_test=getarg_tests/doubledash -Run test_bitcoin --help for the full list. +Run `test_bitcoin --help` for the full list. diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 82d61209b5..fa9624f13d 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -122,6 +122,10 @@ BOOST_AUTO_TEST_CASE(siphash) hasher3.Write(uint64_t(x)|(uint64_t(x+1)<<8)|(uint64_t(x+2)<<16)|(uint64_t(x+3)<<24)| (uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56)); } + + CHashWriter ss(SER_DISK, CLIENT_VERSION); + ss << CTransaction(); + BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index fd581db52e..15fceb963a 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -181,9 +181,7 @@ void TestPackageSelection(const CChainParams& chainparams, CScript scriptPubKey, // NOTE: These tests rely on CreateNewBlock doing its own self-validation! BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) { - // Disable size accounting (CPFP does not support it) - mapArgs["-blockmaxsize"] = strprintf("%u", MAX_BLOCK_SERIALIZED_SIZE); - + // Note that by default, these tests run with size accounting enabled. const CChainParams& chainparams = Params(CBaseChainParams::MAIN); CScript scriptPubKey = CScript() << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f") << OP_CHECKSIG; CBlockTemplate *pblocktemplate; diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 856f9b8423..056f2982cf 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -60,6 +60,11 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha pcoinsdbview = new CCoinsViewDB(1 << 23, true); pcoinsTip = new CCoinsViewCache(pcoinsdbview); InitBlockIndex(chainparams); + { + CValidationState state; + bool ok = ActivateBestChain(state, chainparams); + BOOST_CHECK(ok); + } nScriptCheckThreads = 3; for (int i=0; i < nScriptCheckThreads-1; i++) threadGroup.create_thread(&ThreadScriptCheck); diff --git a/src/tinyformat.h b/src/tinyformat.h index c6ec0419b3..17f0360c42 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -67,7 +67,9 @@ // weekday, month, day, hour, min); // std::cout << date; // -// These are the three primary interface functions. +// These are the three primary interface functions. There is also a +// convenience function printfln() which appends a newline to the usual result +// of printf() for super simple logging. // // // User defined format functions @@ -86,6 +88,18 @@ // defined function bodies, use the macro TINYFORMAT_FOREACH_ARGNUM. For an // example, see the implementation of printf() at the end of the source file. // +// Sometimes it's useful to be able to pass a list of format arguments through +// to a non-template function. The FormatList class is provided as a way to do +// this by storing the argument list in a type-opaque way. Continuing the +// example from above, we construct a FormatList using makeFormatList(): +// +// FormatListRef formatList = tfm::makeFormatList(weekday, month, day, hour, min); +// +// The format list can now be passed into any non-template function and used +// via a call to the vformat() function: +// +// tfm::vformat(std::cout, "%s, %s %d, %.2d:%.2d\n", formatList); +// // // Additional API information // -------------------------- @@ -118,6 +132,7 @@ namespace tfm = tinyformat; //------------------------------------------------------------------------------ // Implementation details. +#include <algorithm> #include <cassert> #include <iostream> #include <sstream> @@ -133,20 +148,20 @@ namespace tfm = tinyformat; # endif #endif -#ifdef __GNUC__ -# define TINYFORMAT_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) -# define TINYFORMAT_NOINLINE __declspec(noinline) -#else -# define TINYFORMAT_NOINLINE -#endif - #if defined(__GLIBCXX__) && __GLIBCXX__ < 20080201 // std::showpos is broken on old libstdc++ as provided with OSX. See // http://gcc.gnu.org/ml/libstdc++/2007-11/msg00075.html # define TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND #endif +#ifdef __APPLE__ +// Workaround OSX linker warning: xcode uses different default symbol +// visibilities for static libs vs executables (see issue #25) +# define TINYFORMAT_HIDDEN __attribute__((visibility("hidden"))) +#else +# define TINYFORMAT_HIDDEN +#endif + namespace tinyformat { //------------------------------------------------------------------------------ @@ -247,6 +262,29 @@ struct convertToInt<T,true> static int invoke(const T& value) { return static_cast<int>(value); } }; +// Format at most ntrunc characters to the given stream. +template<typename T> +inline void formatTruncated(std::ostream& out, const T& value, int ntrunc) +{ + std::ostringstream tmp; + tmp << value; + std::string result = tmp.str(); + out.write(result.c_str(), (std::min)(ntrunc, static_cast<int>(result.size()))); +} +#define TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(type) \ +inline void formatTruncated(std::ostream& out, type* value, int ntrunc) \ +{ \ + std::streamsize len = 0; \ + while(len < ntrunc && value[len] != 0) \ + ++len; \ + out.write(value, len); \ +} +// Overload for const char* and char*. Could overload for signed & unsigned +// char too, but these are technically unneeded for printf compatibility. +TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(const char) +TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(char) +#undef TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR + } // namespace detail @@ -255,18 +293,20 @@ struct convertToInt<T,true> // desired. -// Format a value into a stream. Called from format() for all types by default. -// -// Users may override this for their own types. When this function is called, -// the stream flags will have been modified according to the format string. -// The format specification is provided in the range [fmtBegin, fmtEnd). -// -// By default, formatValue() uses the usual stream insertion operator -// operator<< to format the type T, with special cases for the %c and %p -// conversions. +/// Format a value into a stream, delegating to operator<< by default. +/// +/// Users may override this for their own types. When this function is called, +/// the stream flags will have been modified according to the format string. +/// The format specification is provided in the range [fmtBegin, fmtEnd). For +/// truncating conversions, ntrunc is set to the desired maximum number of +/// characters, for example "%.7s" calls formatValue with ntrunc = 7. +/// +/// By default, formatValue() uses the usual stream insertion operator +/// operator<< to format the type T, with special cases for the %c and %p +/// conversions. template<typename T> inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, - const char* fmtEnd, const T& value) + const char* fmtEnd, int ntrunc, const T& value) { #ifndef TINYFORMAT_ALLOW_WCHAR_STRINGS // Since we don't support printing of wchar_t using "%ls", make it fail at @@ -288,6 +328,12 @@ inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, #ifdef TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND else if(detail::formatZeroIntegerWorkaround<T>::invoke(out, value)) /**/; #endif + else if(ntrunc >= 0) + { + // Take care not to overread C strings in truncating conversions like + // "%.4s" where at most 4 characters may be read. + detail::formatTruncated(out, value, ntrunc); + } else out << value; } @@ -296,7 +342,7 @@ inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, // Overloaded version for char types to support printing as an integer #define TINYFORMAT_DEFINE_FORMATVALUE_CHAR(charType) \ inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, \ - const char* fmtEnd, charType value) \ + const char* fmtEnd, int /**/, charType value) \ { \ switch(*(fmtEnd-1)) \ { \ @@ -435,225 +481,91 @@ cog.outl('#define TINYFORMAT_FOREACH_ARGNUM(m) \\\n ' + namespace detail { -// Class holding current position in format string and an output stream into -// which arguments are formatted. -class FormatIterator +// Type-opaque holder for an argument to format(), with associated actions on +// the type held as explicit function pointers. This allows FormatArg's for +// each argument to be allocated as a homogenous array inside FormatList +// whereas a naive implementation based on inheritance does not. +class FormatArg { public: - // Flags for features not representable with standard stream state - enum ExtraFormatFlags - { - Flag_None = 0, - Flag_TruncateToPrecision = 1<<0, // truncate length to stream precision() - Flag_SpacePadPositive = 1<<1, // pad positive values with spaces - Flag_VariableWidth = 1<<2, // variable field width in arg list - Flag_VariablePrecision = 1<<3 // variable field precision in arg list - }; - - // out is the output stream, fmt is the full format string - FormatIterator(std::ostream& out, const char* fmt) - : m_out(out), - m_fmt(fmt), - m_extraFlags(Flag_None), - m_wantWidth(false), - m_wantPrecision(false), - m_variableWidth(0), - m_variablePrecision(0), - m_origWidth(out.width()), - m_origPrecision(out.precision()), - m_origFlags(out.flags()), - m_origFill(out.fill()) + FormatArg() {} + + template<typename T> + FormatArg(const T& value) + : m_value(static_cast<const void*>(&value)), + m_formatImpl(&formatImpl<T>), + m_toIntImpl(&toIntImpl<T>) { } - // Print remaining part of format string. - void finish() + void format(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc) const { - // It would be nice if we could do this from the destructor, but we - // can't if TINFORMAT_ERROR is used to throw an exception! - m_fmt = printFormatStringLiteral(m_out, m_fmt); - if(*m_fmt != '\0') - TINYFORMAT_ERROR("tinyformat: Too many conversion specifiers in format string"); + m_formatImpl(out, fmtBegin, fmtEnd, ntrunc, m_value); } - ~FormatIterator() + int toInt() const { - // Restore stream state - m_out.width(m_origWidth); - m_out.precision(m_origPrecision); - m_out.flags(m_origFlags); - m_out.fill(m_origFill); + return m_toIntImpl(m_value); } - template<typename T> - void accept(const T& value); - private: - // Parse and return an integer from the string c, as atoi() - // On return, c is set to one past the end of the integer. - static int parseIntAndAdvance(const char*& c) + template<typename T> + TINYFORMAT_HIDDEN static void formatImpl(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc, const void* value) { - int i = 0; - for(;*c >= '0' && *c <= '9'; ++c) - i = 10*i + (*c - '0'); - return i; + formatValue(out, fmtBegin, fmtEnd, ntrunc, *static_cast<const T*>(value)); } - // Format at most truncLen characters of a C string to the given - // stream. Return true if formatting proceeded (generic version always - // returns false) template<typename T> - static bool formatCStringTruncate(std::ostream& /*out*/, const T& /*value*/, - std::streamsize /*truncLen*/) + TINYFORMAT_HIDDEN static int toIntImpl(const void* value) { - return false; - } -# define TINYFORMAT_DEFINE_FORMAT_C_STRING_TRUNCATE(type) \ - static bool formatCStringTruncate(std::ostream& out, type* value, \ - std::streamsize truncLen) \ - { \ - std::streamsize len = 0; \ - while(len < truncLen && value[len] != 0) \ - ++len; \ - out.write(value, len); \ - return true; \ - } - // Overload for const char* and char*. Could overload for signed & - // unsigned char too, but these are technically unneeded for printf - // compatibility. - TINYFORMAT_DEFINE_FORMAT_C_STRING_TRUNCATE(const char) - TINYFORMAT_DEFINE_FORMAT_C_STRING_TRUNCATE(char) -# undef TINYFORMAT_DEFINE_FORMAT_C_STRING_TRUNCATE - - // Print literal part of format string and return next format spec - // position. - // - // Skips over any occurrences of '%%', printing a literal '%' to the - // output. The position of the first % character of the next - // nontrivial format spec is returned, or the end of string. - static const char* printFormatStringLiteral(std::ostream& out, - const char* fmt) - { - const char* c = fmt; - for(; true; ++c) - { - switch(*c) - { - case '\0': - out.write(fmt, static_cast<std::streamsize>(c - fmt)); - return c; - case '%': - out.write(fmt, static_cast<std::streamsize>(c - fmt)); - if(*(c+1) != '%') - return c; - // for "%%", tack trailing % onto next literal section. - fmt = ++c; - break; - } - } + return convertToInt<T>::invoke(*static_cast<const T*>(value)); } - static const char* streamStateFromFormat(std::ostream& out, - unsigned int& extraFlags, - const char* fmtStart, - int variableWidth, - int variablePrecision); - - // Private copy & assign: Kill gcc warnings with -Weffc++ - FormatIterator(const FormatIterator&); - FormatIterator& operator=(const FormatIterator&); - - // Stream, current format string & state - std::ostream& m_out; - const char* m_fmt; - unsigned int m_extraFlags; - // State machine info for handling of variable width & precision - bool m_wantWidth; - bool m_wantPrecision; - int m_variableWidth; - int m_variablePrecision; - // Saved stream state - std::streamsize m_origWidth; - std::streamsize m_origPrecision; - std::ios::fmtflags m_origFlags; - char m_origFill; + const void* m_value; + void (*m_formatImpl)(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc, const void* value); + int (*m_toIntImpl)(const void* value); }; -// Accept a value for formatting into the internal stream. -template<typename T> -TINYFORMAT_NOINLINE // < greatly reduces bloat in optimized builds -void FormatIterator::accept(const T& value) +// Parse and return an integer from the string c, as atoi() +// On return, c is set to one past the end of the integer. +inline int parseIntAndAdvance(const char*& c) { - // Parse the format string - const char* fmtEnd = 0; - if(m_extraFlags == Flag_None && !m_wantWidth && !m_wantPrecision) - { - m_fmt = printFormatStringLiteral(m_out, m_fmt); - fmtEnd = streamStateFromFormat(m_out, m_extraFlags, m_fmt, 0, 0); - m_wantWidth = (m_extraFlags & Flag_VariableWidth) != 0; - m_wantPrecision = (m_extraFlags & Flag_VariablePrecision) != 0; - } - // Consume value as variable width and precision specifier if necessary - if(m_extraFlags & (Flag_VariableWidth | Flag_VariablePrecision)) - { - if(m_wantWidth || m_wantPrecision) - { - int v = convertToInt<T>::invoke(value); - if(m_wantWidth) - { - m_variableWidth = v; - m_wantWidth = false; - } - else if(m_wantPrecision) - { - m_variablePrecision = v; - m_wantPrecision = false; - } - return; - } - // If we get here, we've set both the variable precision and width as - // required and we need to rerun the stream state setup to insert these. - fmtEnd = streamStateFromFormat(m_out, m_extraFlags, m_fmt, - m_variableWidth, m_variablePrecision); - } + int i = 0; + for(;*c >= '0' && *c <= '9'; ++c) + i = 10*i + (*c - '0'); + return i; +} - // Format the value into the stream. - if(!(m_extraFlags & (Flag_SpacePadPositive | Flag_TruncateToPrecision))) - formatValue(m_out, m_fmt, fmtEnd, value); - else +// Print literal part of format string and return next format spec +// position. +// +// Skips over any occurrences of '%%', printing a literal '%' to the +// output. The position of the first % character of the next +// nontrivial format spec is returned, or the end of string. +inline const char* printFormatStringLiteral(std::ostream& out, const char* fmt) +{ + const char* c = fmt; + for(;; ++c) { - // The following are special cases where there's no direct - // correspondence between stream formatting and the printf() behaviour. - // Instead, we simulate the behaviour crudely by formatting into a - // temporary string stream and munging the resulting string. - std::ostringstream tmpStream; - tmpStream.copyfmt(m_out); - if(m_extraFlags & Flag_SpacePadPositive) - tmpStream.setf(std::ios::showpos); - // formatCStringTruncate is required for truncating conversions like - // "%.4s" where at most 4 characters of the c-string should be read. - // If we didn't include this special case, we might read off the end. - if(!( (m_extraFlags & Flag_TruncateToPrecision) && - formatCStringTruncate(tmpStream, value, m_out.precision()) )) - { - // Not a truncated c-string; just format normally. - formatValue(tmpStream, m_fmt, fmtEnd, value); - } - std::string result = tmpStream.str(); // allocates... yuck. - if(m_extraFlags & Flag_SpacePadPositive) + switch(*c) { - for(size_t i = 0, iend = result.size(); i < iend; ++i) - if(result[i] == '+') - result[i] = ' '; + case '\0': + out.write(fmt, c - fmt); + return c; + case '%': + out.write(fmt, c - fmt); + if(*(c+1) != '%') + return c; + // for "%%", tack trailing % onto next literal section. + fmt = ++c; + break; + default: + break; } - if((m_extraFlags & Flag_TruncateToPrecision) && - (int)result.size() > (int)m_out.precision()) - m_out.write(result.c_str(), m_out.precision()); - else - m_out << result; } - m_extraFlags = Flag_None; - m_fmt = fmtEnd; } @@ -663,13 +575,14 @@ void FormatIterator::accept(const T& value) // with the form "%[flags][width][.precision][length]type". // // Formatting options which can't be natively represented using the ostream -// state are returned in the extraFlags parameter which is a bitwise -// combination of values from the ExtraFormatFlags enum. -inline const char* FormatIterator::streamStateFromFormat(std::ostream& out, - unsigned int& extraFlags, - const char* fmtStart, - int variableWidth, - int variablePrecision) +// state are returned in spacePadPositive (for space padded positive numbers) +// and ntrunc (for truncating conversions). argIndex is incremented if +// necessary to pull out variable width and precision . The function returns a +// pointer to the character after the end of the current format spec. +inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositive, + int& ntrunc, const char* fmtStart, + const detail::FormatArg* formatters, + int& argIndex, int numFormatters) { if(*fmtStart != '%') { @@ -684,9 +597,9 @@ inline const char* FormatIterator::streamStateFromFormat(std::ostream& out, out.unsetf(std::ios::adjustfield | std::ios::basefield | std::ios::floatfield | std::ios::showbase | std::ios::boolalpha | std::ios::showpoint | std::ios::showpos | std::ios::uppercase); - extraFlags = Flag_None; bool precisionSet = false; bool widthSet = false; + int widthExtra = 0; const char* c = fmtStart + 1; // 1) Parse flags for(;; ++c) @@ -713,12 +626,15 @@ inline const char* FormatIterator::streamStateFromFormat(std::ostream& out, case ' ': // overridden by show positive sign, '+' flag. if(!(out.flags() & std::ios::showpos)) - extraFlags |= Flag_SpacePadPositive; + spacePadPositive = true; continue; case '+': out.setf(std::ios::showpos); - extraFlags &= ~Flag_SpacePadPositive; + spacePadPositive = false; + widthExtra = 1; continue; + default: + break; } break; } @@ -731,15 +647,19 @@ inline const char* FormatIterator::streamStateFromFormat(std::ostream& out, if(*c == '*') { widthSet = true; - if(variableWidth < 0) + int width = 0; + if(argIndex < numFormatters) + width = formatters[argIndex++].toInt(); + else + TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable width"); + if(width < 0) { // negative widths correspond to '-' flag set out.fill(' '); out.setf(std::ios::left, std::ios::adjustfield); - variableWidth = -variableWidth; + width = -width; } - out.width(variableWidth); - extraFlags |= Flag_VariableWidth; + out.width(width); ++c; } // 3) Parse precision @@ -750,8 +670,10 @@ inline const char* FormatIterator::streamStateFromFormat(std::ostream& out, if(*c == '*') { ++c; - extraFlags |= Flag_VariablePrecision; - precision = variablePrecision; + if(argIndex < numFormatters) + precision = formatters[argIndex++].toInt(); + else + TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable precision"); } else { @@ -814,7 +736,7 @@ inline const char* FormatIterator::streamStateFromFormat(std::ostream& out, break; case 's': if(precisionSet) - extraFlags |= Flag_TruncateToPrecision; + ntrunc = static_cast<int>(out.precision()); // Make %s print booleans as "true" and "false" out.setf(std::ios::boolalpha); break; @@ -826,6 +748,8 @@ inline const char* FormatIterator::streamStateFromFormat(std::ostream& out, TINYFORMAT_ERROR("tinyformat: Conversion spec incorrectly " "terminated by end of string"); return c; + default: + break; } if(intConversion && precisionSet && !widthSet) { @@ -833,7 +757,7 @@ inline const char* FormatIterator::streamStateFromFormat(std::ostream& out, // padded with zeros on the left). This isn't really supported by the // iostreams, but we can approximately simulate it with the width if // the width isn't otherwise used. - out.width(out.precision()); + out.width(out.precision() + widthExtra); out.setf(std::ios::internal, std::ios::adjustfield); out.fill('0'); } @@ -841,170 +765,282 @@ inline const char* FormatIterator::streamStateFromFormat(std::ostream& out, } - //------------------------------------------------------------------------------ -// Private format function on top of which the public interface is implemented. -// We enforce a mimimum of one value to be formatted to prevent bugs looking like -// -// const char* myStr = "100% broken"; -// printf(myStr); // Parses % as a format specifier -#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES - -template<typename T1> -void format(FormatIterator& fmtIter, const T1& value1) +inline void formatImpl(std::ostream& out, const char* fmt, + const detail::FormatArg* formatters, + int numFormatters) { - fmtIter.accept(value1); - fmtIter.finish(); + // Saved stream state + std::streamsize origWidth = out.width(); + std::streamsize origPrecision = out.precision(); + std::ios::fmtflags origFlags = out.flags(); + char origFill = out.fill(); + + for (int argIndex = 0; argIndex < numFormatters; ++argIndex) + { + // Parse the format string + fmt = printFormatStringLiteral(out, fmt); + bool spacePadPositive = false; + int ntrunc = -1; + const char* fmtEnd = streamStateFromFormat(out, spacePadPositive, ntrunc, fmt, + formatters, argIndex, numFormatters); + if (argIndex >= numFormatters) + { + // Check args remain after reading any variable width/precision + TINYFORMAT_ERROR("tinyformat: Not enough format arguments"); + return; + } + const FormatArg& arg = formatters[argIndex]; + // Format the arg into the stream. + if(!spacePadPositive) + arg.format(out, fmt, fmtEnd, ntrunc); + else + { + // The following is a special case with no direct correspondence + // between stream formatting and the printf() behaviour. Simulate + // it crudely by formatting into a temporary string stream and + // munging the resulting string. + std::ostringstream tmpStream; + tmpStream.copyfmt(out); + tmpStream.setf(std::ios::showpos); + arg.format(tmpStream, fmt, fmtEnd, ntrunc); + std::string result = tmpStream.str(); // allocates... yuck. + for(size_t i = 0, iend = result.size(); i < iend; ++i) + if(result[i] == '+') result[i] = ' '; + out << result; + } + fmt = fmtEnd; + } + + // Print remaining part of format string. + fmt = printFormatStringLiteral(out, fmt); + if(*fmt != '\0') + TINYFORMAT_ERROR("tinyformat: Too many conversion specifiers in format string"); + + // Restore stream state + out.width(origWidth); + out.precision(origPrecision); + out.flags(origFlags); + out.fill(origFill); } -// General version for C++11 -template<typename T1, typename... Args> -void format(FormatIterator& fmtIter, const T1& value1, const Args&... args) +} // namespace detail + + +/// List of template arguments format(), held in a type-opaque way. +/// +/// A const reference to FormatList (typedef'd as FormatListRef) may be +/// conveniently used to pass arguments to non-template functions: All type +/// information has been stripped from the arguments, leaving just enough of a +/// common interface to perform formatting as required. +class FormatList { - fmtIter.accept(value1); - format(fmtIter, args...); -} + public: + FormatList(detail::FormatArg* formatters, int N) + : m_formatters(formatters), m_N(N) { } -#else + friend void vformat(std::ostream& out, const char* fmt, + const FormatList& list); -inline void format(FormatIterator& fmtIter) + private: + const detail::FormatArg* m_formatters; + int m_N; +}; + +/// Reference to type-opaque format list for passing to vformat() +typedef const FormatList& FormatListRef; + + +namespace detail { + +// Format list subclass with fixed storage to avoid dynamic allocation +template<int N> +class FormatListN : public FormatList { - fmtIter.finish(); -} + public: +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + template<typename... Args> + FormatListN(const Args&... args) + : FormatList(&m_formatterStore[0], N), + m_formatterStore { FormatArg(args)... } + { static_assert(sizeof...(args) == N, "Number of args must be N"); } +#else // C++98 version + void init(int) {} +# define TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR(n) \ + \ + template<TINYFORMAT_ARGTYPES(n)> \ + FormatListN(TINYFORMAT_VARARGS(n)) \ + : FormatList(&m_formatterStore[0], n) \ + { assert(n == N); init(0, TINYFORMAT_PASSARGS(n)); } \ + \ + template<TINYFORMAT_ARGTYPES(n)> \ + void init(int i, TINYFORMAT_VARARGS(n)) \ + { \ + m_formatterStore[i] = FormatArg(v1); \ + init(i+1 TINYFORMAT_PASSARGS_TAIL(n)); \ + } -// General version for C++98 -#define TINYFORMAT_MAKE_FORMAT_DETAIL(n) \ -template<TINYFORMAT_ARGTYPES(n)> \ -void format(detail::FormatIterator& fmtIter, TINYFORMAT_VARARGS(n)) \ -{ \ - fmtIter.accept(v1); \ - format(fmtIter TINYFORMAT_PASSARGS_TAIL(n)); \ -} + TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR) +# undef TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR +#endif -TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMAT_DETAIL) -#undef TINYFORMAT_MAKE_FORMAT_DETAIL + private: + FormatArg m_formatterStore[N]; +}; -#endif // End C++98 variadic template emulation for format() +// Special 0-arg version - MSVC says zero-sized C array in struct is nonstandard +template<> class FormatListN<0> : public FormatList +{ + public: FormatListN() : FormatList(0, 0) {} +}; } // namespace detail //------------------------------------------------------------------------------ -// Implement all the main interface functions here in terms of detail::format() +// Primary API functions #ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES -// C++11 - the simple case -template<typename T1, typename... Args> -void format(std::ostream& out, const char* fmt, const T1& v1, const Args&... args) +/// Make type-agnostic format list from list of template arguments. +/// +/// The exact return type of this function is an implementation detail and +/// shouldn't be relied upon. Instead it should be stored as a FormatListRef: +/// +/// FormatListRef formatList = makeFormatList( /*...*/ ); +template<typename... Args> +detail::FormatListN<sizeof...(Args)> makeFormatList(const Args&... args) { - detail::FormatIterator fmtIter(out, fmt); - format(fmtIter, v1, args...); + return detail::FormatListN<sizeof...(args)>(args...); } -template<typename T1, typename... Args> -std::string format(const char* fmt, const T1& v1, const Args&... args) +#else // C++98 version + +inline detail::FormatListN<0> makeFormatList() +{ + return detail::FormatListN<0>(); +} +#define TINYFORMAT_MAKE_MAKEFORMATLIST(n) \ +template<TINYFORMAT_ARGTYPES(n)> \ +detail::FormatListN<n> makeFormatList(TINYFORMAT_VARARGS(n)) \ +{ \ + return detail::FormatListN<n>(TINYFORMAT_PASSARGS(n)); \ +} +TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_MAKEFORMATLIST) +#undef TINYFORMAT_MAKE_MAKEFORMATLIST + +#endif + +/// Format list of arguments to the stream according to the given format string. +/// +/// The name vformat() is chosen for the semantic similarity to vprintf(): the +/// list of format arguments is held in a single function argument. +inline void vformat(std::ostream& out, const char* fmt, FormatListRef list) +{ + detail::formatImpl(out, fmt, list.m_formatters, list.m_N); +} + + +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + +/// Format list of arguments to the stream according to given format string. +template<typename... Args> +void format(std::ostream& out, const char* fmt, const Args&... args) +{ + vformat(out, fmt, makeFormatList(args...)); +} + +/// Format list of arguments according to the given format string and return +/// the result as a string. +template<typename... Args> +std::string format(const char* fmt, const Args&... args) { std::ostringstream oss; - format(oss, fmt, v1, args...); + format(oss, fmt, args...); return oss.str(); } -template<typename T1, typename... Args> -std::string format(const std::string &fmt, const T1& v1, const Args&... args) +/// Format list of arguments to std::cout, according to the given format string +template<typename... Args> +void printf(const char* fmt, const Args&... args) +{ + format(std::cout, fmt, args...); +} + +template<typename... Args> +void printfln(const char* fmt, const Args&... args) +{ + format(std::cout, fmt, args...); + std::cout << '\n'; +} + +#else // C++98 version + +inline void format(std::ostream& out, const char* fmt) +{ + vformat(out, fmt, makeFormatList()); +} + +inline std::string format(const char* fmt) { std::ostringstream oss; - format(oss, fmt.c_str(), v1, args...); + format(oss, fmt); return oss.str(); } -template<typename T1, typename... Args> -void printf(const char* fmt, const T1& v1, const Args&... args) +inline void printf(const char* fmt) { - format(std::cout, fmt, v1, args...); + format(std::cout, fmt); } -#else +inline void printfln(const char* fmt) +{ + format(std::cout, fmt); + std::cout << '\n'; +} -// C++98 - define the interface functions using the wrapping macros #define TINYFORMAT_MAKE_FORMAT_FUNCS(n) \ \ template<TINYFORMAT_ARGTYPES(n)> \ void format(std::ostream& out, const char* fmt, TINYFORMAT_VARARGS(n)) \ { \ - tinyformat::detail::FormatIterator fmtIter(out, fmt); \ - tinyformat::detail::format(fmtIter, TINYFORMAT_PASSARGS(n)); \ + vformat(out, fmt, makeFormatList(TINYFORMAT_PASSARGS(n))); \ } \ \ template<TINYFORMAT_ARGTYPES(n)> \ std::string format(const char* fmt, TINYFORMAT_VARARGS(n)) \ { \ std::ostringstream oss; \ - tinyformat::format(oss, fmt, TINYFORMAT_PASSARGS(n)); \ + format(oss, fmt, TINYFORMAT_PASSARGS(n)); \ return oss.str(); \ } \ \ template<TINYFORMAT_ARGTYPES(n)> \ -std::string format(const std::string &fmt, TINYFORMAT_VARARGS(n)) \ +void printf(const char* fmt, TINYFORMAT_VARARGS(n)) \ { \ - std::ostringstream oss; \ - tinyformat::format(oss, fmt.c_str(), TINYFORMAT_PASSARGS(n)); \ - return oss.str(); \ + format(std::cout, fmt, TINYFORMAT_PASSARGS(n)); \ } \ \ template<TINYFORMAT_ARGTYPES(n)> \ -void printf(const char* fmt, TINYFORMAT_VARARGS(n)) \ +void printfln(const char* fmt, TINYFORMAT_VARARGS(n)) \ { \ - tinyformat::format(std::cout, fmt, TINYFORMAT_PASSARGS(n)); \ + format(std::cout, fmt, TINYFORMAT_PASSARGS(n)); \ + std::cout << '\n'; \ } TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMAT_FUNCS) #undef TINYFORMAT_MAKE_FORMAT_FUNCS -#endif - -//------------------------------------------------------------------------------ -// Define deprecated wrapping macro for backward compatibility in tinyformat -// 1.x. Will be removed in version 2! -#define TINYFORMAT_WRAP_FORMAT_EXTRA_ARGS -#define TINYFORMAT_WRAP_FORMAT_N(n, returnType, funcName, funcDeclSuffix, \ - bodyPrefix, streamName, bodySuffix) \ -template<TINYFORMAT_ARGTYPES(n)> \ -returnType funcName(TINYFORMAT_WRAP_FORMAT_EXTRA_ARGS const char* fmt, \ - TINYFORMAT_VARARGS(n)) funcDeclSuffix \ -{ \ - bodyPrefix \ - tinyformat::format(streamName, fmt, TINYFORMAT_PASSARGS(n)); \ - bodySuffix \ -} \ - -#define TINYFORMAT_WRAP_FORMAT(returnType, funcName, funcDeclSuffix, \ - bodyPrefix, streamName, bodySuffix) \ -inline \ -returnType funcName(TINYFORMAT_WRAP_FORMAT_EXTRA_ARGS const char* fmt \ - ) funcDeclSuffix \ -{ \ - bodyPrefix \ - tinyformat::detail::FormatIterator(streamName, fmt).finish(); \ - bodySuffix \ -} \ -TINYFORMAT_WRAP_FORMAT_N(1 , returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(2 , returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(3 , returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(4 , returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(5 , returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(6 , returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(7 , returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(8 , returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(9 , returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(10, returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(11, returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(12, returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(13, returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(14, returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(15, returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ -TINYFORMAT_WRAP_FORMAT_N(16, returnType, funcName, funcDeclSuffix, bodyPrefix, streamName, bodySuffix) \ +#endif +// Added for Bitcoin Core +template<typename... Args> +std::string format(const std::string &fmt, const Args&... args) +{ + std::ostringstream oss; + format(oss, fmt.c_str(), args...); + return oss.str(); +} } // namespace tinyformat diff --git a/src/txmempool.cpp b/src/txmempool.cpp index a48a6d9465..b631c48484 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -28,7 +28,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransaction& _tx, const CAmount& _nFee, hadNoDependencies(poolHasNoInputsOf), inChainInputValue(_inChainInputValue), spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) { - nTxCost = GetTransactionCost(_tx); + nTxWeight = GetTransactionWeight(_tx); nModSize = _tx.CalculateModifiedSize(GetTxSize()); nUsageSize = RecursiveDynamicUsage(*tx) + memusage::DynamicUsage(tx); @@ -75,7 +75,7 @@ void CTxMemPoolEntry::UpdateLockPoints(const LockPoints& lp) size_t CTxMemPoolEntry::GetTxSize() const { - return GetVirtualTransactionSize(nTxCost); + return GetVirtualTransactionSize(nTxWeight, sigOpCost); } // Update the given tx for any in-mempool descendants. @@ -657,6 +657,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate(const_cast<CCoinsViewCache*>(pcoins)); + const int64_t nSpendHeight = GetSpendHeight(mempoolDuplicate); LOCK(cs); list<const CTxMemPoolEntry*> waitingOnDependants; @@ -737,7 +738,9 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const waitingOnDependants.push_back(&(*it)); else { CValidationState state; - assert(CheckInputs(tx, state, mempoolDuplicate, false, 0, false, NULL)); + bool fCheckResult = tx.IsCoinBase() || + Consensus::CheckTxInputs(tx, state, mempoolDuplicate, nSpendHeight); + assert(fCheckResult); UpdateCoins(tx, mempoolDuplicate, 1000000); } } @@ -751,7 +754,9 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const stepsSinceLastRemove++; assert(stepsSinceLastRemove < waitingOnDependants.size()); } else { - assert(CheckInputs(entry->GetTx(), state, mempoolDuplicate, false, 0, false, NULL)); + bool fCheckResult = entry->GetTx().IsCoinBase() || + Consensus::CheckTxInputs(entry->GetTx(), state, mempoolDuplicate, nSpendHeight); + assert(fCheckResult); UpdateCoins(entry->GetTx(), mempoolDuplicate, 1000000); stepsSinceLastRemove = 0; } diff --git a/src/txmempool.h b/src/txmempool.h index e5a500e19d..2c2127f326 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -78,7 +78,7 @@ class CTxMemPoolEntry private: std::shared_ptr<const CTransaction> tx; CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups - size_t nTxCost; //!< ... and avoid recomputing tx cost (also used for GetTxSize()) + size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) size_t nModSize; //!< ... and modified size for priority size_t nUsageSize; //!< ... and total memory usage int64_t nTime; //!< Local time when entering the mempool @@ -122,7 +122,7 @@ public: double GetPriority(unsigned int currentHeight) const; const CAmount& GetFee() const { return nFee; } size_t GetTxSize() const; - size_t GetTxCost() const { return nTxCost; } + size_t GetTxWeight() const { return nTxWeight; } int64_t GetTime() const { return nTime; } unsigned int GetHeight() const { return entryHeight; } bool WasClearAtEntry() const { return hadNoDependencies; } diff --git a/src/util.h b/src/util.h index ac4b947785..39328b51ef 100644 --- a/src/util.h +++ b/src/util.h @@ -77,33 +77,17 @@ int LogPrintStr(const std::string &str); #define LogPrintf(...) LogPrint(NULL, __VA_ARGS__) -template<typename T1, typename... Args> -static inline int LogPrint(const char* category, const char* fmt, const T1& v1, const Args&... args) +template<typename... Args> +static inline int LogPrint(const char* category, const char* fmt, const Args&... args) { if(!LogAcceptCategory(category)) return 0; \ - return LogPrintStr(tfm::format(fmt, v1, args...)); + return LogPrintStr(tfm::format(fmt, args...)); } -template<typename T1, typename... Args> -bool error(const char* fmt, const T1& v1, const Args&... args) +template<typename... Args> +bool error(const char* fmt, const Args&... args) { - LogPrintStr("ERROR: " + tfm::format(fmt, v1, args...) + "\n"); - return false; -} - -/** - * Zero-arg versions of logging and error, these are not covered by - * the variadic templates above (and don't take format arguments but - * bare strings). - */ -static inline int LogPrint(const char* category, const char* s) -{ - if(!LogAcceptCategory(category)) return 0; - return LogPrintStr(s); -} -static inline bool error(const char* s) -{ - LogPrintStr(std::string("ERROR: ") + s + "\n"); + LogPrintStr("ERROR: " + tfm::format(fmt, args...) + "\n"); return false; } diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index d55cc68dc0..fe8b53ceb0 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -309,8 +309,7 @@ UniValue importprunedfunds(const UniValue& params, bool fHelp) LOCK2(cs_main, pwalletMain->cs_wallet); if (pwalletMain->IsMine(tx)) { - CWalletDB walletdb(pwalletMain->strWalletFile, "r+", false); - pwalletMain->AddToWallet(wtx, false, &walletdb); + pwalletMain->AddToWallet(wtx, false); return NullUniValue; } @@ -602,19 +601,42 @@ UniValue dumpwallet(const UniValue& params, bool fHelp) file << strprintf("# * Best block at time of backup was %i (%s),\n", chainActive.Height(), chainActive.Tip()->GetBlockHash().ToString()); file << strprintf("# mined on %s\n", EncodeDumpTime(chainActive.Tip()->GetBlockTime())); file << "\n"; + + // add the base58check encoded extended master if the wallet uses HD + CKeyID masterKeyID = pwalletMain->GetHDChain().masterKeyID; + if (!masterKeyID.IsNull()) + { + CKey key; + if (pwalletMain->GetKey(masterKeyID, key)) + { + CExtKey masterKey; + masterKey.SetMaster(key.begin(), key.size()); + + CBitcoinExtKey b58extkey; + b58extkey.SetKey(masterKey); + + file << "# extended private masterkey: " << b58extkey.ToString() << "\n\n"; + } + } for (std::vector<std::pair<int64_t, CKeyID> >::const_iterator it = vKeyBirth.begin(); it != vKeyBirth.end(); it++) { const CKeyID &keyid = it->second; std::string strTime = EncodeDumpTime(it->first); std::string strAddr = CBitcoinAddress(keyid).ToString(); CKey key; if (pwalletMain->GetKey(keyid, key)) { + file << strprintf("%s %s ", CBitcoinSecret(key).ToString(), strTime); if (pwalletMain->mapAddressBook.count(keyid)) { - file << strprintf("%s %s label=%s # addr=%s\n", CBitcoinSecret(key).ToString(), strTime, EncodeDumpString(pwalletMain->mapAddressBook[keyid].name), strAddr); + file << strprintf("label=%s", EncodeDumpString(pwalletMain->mapAddressBook[keyid].name)); + } else if (keyid == masterKeyID) { + file << "hdmaster=1"; } else if (setKeyPool.count(keyid)) { - file << strprintf("%s %s reserve=1 # addr=%s\n", CBitcoinSecret(key).ToString(), strTime, strAddr); + file << "reserve=1"; + } else if (pwalletMain->mapKeyMetadata[keyid].hdKeypath == "m") { + file << "inactivehdmaster=1"; } else { - file << strprintf("%s %s change=1 # addr=%s\n", CBitcoinSecret(key).ToString(), strTime, strAddr); + file << "change=1"; } + file << strprintf(" # addr=%s%s\n", strAddr, (pwalletMain->mapKeyMetadata[keyid].hdKeypath.size() > 0 ? " hdkeypath="+pwalletMain->mapKeyMetadata[keyid].hdKeypath : "")); } } file << "\n"; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index b4831ad795..a90807e514 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2081,7 +2081,7 @@ UniValue encryptwallet(const UniValue& params, bool fHelp) // slack space in .dat files; that is bad if the old data is // unencrypted private keys. So: StartShutdown(); - return "wallet encrypted; Bitcoin server stopping, restart to run with encrypted wallet. The keypool has been flushed, you need to make a new backup."; + return "wallet encrypted; Bitcoin server stopping, restart to run with encrypted wallet. The keypool has been flushed and a new HD seed was generated (if you are using HD). You need to make a new backup."; } UniValue lockunspent(const UniValue& params, bool fHelp) @@ -2260,16 +2260,16 @@ UniValue getwalletinfo(const UniValue& params, bool fHelp) "Returns an object containing various wallet state info.\n" "\nResult:\n" "{\n" - " \"walletversion\": xxxxx, (numeric) the wallet version\n" - " \"balance\": xxxxxxx, (numeric) the total confirmed balance of the wallet in " + CURRENCY_UNIT + "\n" - " \"unconfirmed_balance\": xxx, (numeric) the total unconfirmed balance of the wallet in " + CURRENCY_UNIT + "\n" - " \"immature_balance\": xxxxxx, (numeric) the total immature balance of the wallet in " + CURRENCY_UNIT + "\n" - " \"txcount\": xxxxxxx, (numeric) the total number of transactions in the wallet\n" - " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since GMT epoch) of the oldest pre-generated key in the key pool\n" - " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" - " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" - " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" - " \"masterkeyid\": \"<hash160>\", (string) the Hash160 of the HD master pubkey\n" + " \"walletversion\": xxxxx, (numeric) the wallet version\n" + " \"balance\": xxxxxxx, (numeric) the total confirmed balance of the wallet in " + CURRENCY_UNIT + "\n" + " \"unconfirmed_balance\": xxx, (numeric) the total unconfirmed balance of the wallet in " + CURRENCY_UNIT + "\n" + " \"immature_balance\": xxxxxx, (numeric) the total immature balance of the wallet in " + CURRENCY_UNIT + "\n" + " \"txcount\": xxxxxxx, (numeric) the total number of transactions in the wallet\n" + " \"keypoololdest\": xxxxxx, (numeric) the timestamp (seconds since GMT epoch) of the oldest pre-generated key in the key pool\n" + " \"keypoolsize\": xxxx, (numeric) how many new keys are pre-generated\n" + " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" + " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" + " \"hdmasterkeyid\": \"<hash160>\", (string) the Hash160 of the HD master pubkey\n" "}\n" "\nExamples:\n" + HelpExampleCli("getwalletinfo", "") @@ -2291,7 +2291,7 @@ UniValue getwalletinfo(const UniValue& params, bool fHelp) obj.push_back(Pair("paytxfee", ValueFromAmount(payTxFee.GetFeePerK()))); CKeyID masterKeyID = pwalletMain->GetHDChain().masterKeyID; if (!masterKeyID.IsNull()) - obj.push_back(Pair("masterkeyid", masterKeyID.GetHex())); + obj.push_back(Pair("hdmasterkeyid", masterKeyID.GetHex())); return obj; } diff --git a/src/wallet/test/accounting_tests.cpp b/src/wallet/test/accounting_tests.cpp index d075b2b641..a6cada46a2 100644 --- a/src/wallet/test/accounting_tests.cpp +++ b/src/wallet/test/accounting_tests.cpp @@ -48,7 +48,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) pwalletMain->AddAccountingEntry(ae, walletdb); wtx.mapValue["comment"] = "z"; - pwalletMain->AddToWallet(wtx, false, &walletdb); + pwalletMain->AddToWallet(wtx); vpwtx.push_back(&pwalletMain->mapWallet[wtx.GetHash()]); vpwtx[0]->nTimeReceived = (unsigned int)1333333335; vpwtx[0]->nOrderPos = -1; @@ -90,7 +90,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) --tx.nLockTime; // Just to change the hash :) *static_cast<CTransaction*>(&wtx) = CTransaction(tx); } - pwalletMain->AddToWallet(wtx, false, &walletdb); + pwalletMain->AddToWallet(wtx); vpwtx.push_back(&pwalletMain->mapWallet[wtx.GetHash()]); vpwtx[1]->nTimeReceived = (unsigned int)1333333336; @@ -100,7 +100,7 @@ BOOST_AUTO_TEST_CASE(acc_orderupgrade) --tx.nLockTime; // Just to change the hash :) *static_cast<CTransaction*>(&wtx) = CTransaction(tx); } - pwalletMain->AddToWallet(wtx, false, &walletdb); + pwalletMain->AddToWallet(wtx); vpwtx.push_back(&pwalletMain->mapWallet[wtx.GetHash()]); vpwtx[2]->nTimeReceived = (unsigned int)1333333329; vpwtx[2]->nOrderPos = -1; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 46ed542158..ee9254050c 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -626,6 +626,15 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) Lock(); Unlock(strWalletPassphrase); + + // if we are using HD, replace the HD master key (seed) with a new one + if (!hdChain.masterKeyID.IsNull()) { + CKey key; + CPubKey masterPubKey = GenerateNewHDMasterKey(); + if (!SetHDMasterKey(masterPubKey)) + return false; + } + NewKeyPool(); Lock(); @@ -732,138 +741,143 @@ void CWallet::MarkDirty() } } -bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFromLoadWallet, CWalletDB* pwalletdb) +bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) { + LOCK(cs_wallet); + + CWalletDB walletdb(strWalletFile, "r+", fFlushOnClose); + uint256 hash = wtxIn.GetHash(); - if (fFromLoadWallet) + // Inserts only if not already there, returns tx inserted or tx found + pair<map<uint256, CWalletTx>::iterator, bool> ret = mapWallet.insert(make_pair(hash, wtxIn)); + CWalletTx& wtx = (*ret.first).second; + wtx.BindWallet(this); + bool fInsertedNew = ret.second; + if (fInsertedNew) { - mapWallet[hash] = wtxIn; - CWalletTx& wtx = mapWallet[hash]; - wtx.BindWallet(this); + wtx.nTimeReceived = GetAdjustedTime(); + wtx.nOrderPos = IncOrderPosNext(&walletdb); wtxOrdered.insert(make_pair(wtx.nOrderPos, TxPair(&wtx, (CAccountingEntry*)0))); - AddToSpends(hash); - BOOST_FOREACH(const CTxIn& txin, wtx.vin) { - if (mapWallet.count(txin.prevout.hash)) { - CWalletTx& prevtx = mapWallet[txin.prevout.hash]; - if (prevtx.nIndex == -1 && !prevtx.hashUnset()) { - MarkConflicted(prevtx.hashBlock, wtx.GetHash()); - } - } - } - } - else - { - LOCK(cs_wallet); - // Inserts only if not already there, returns tx inserted or tx found - pair<map<uint256, CWalletTx>::iterator, bool> ret = mapWallet.insert(make_pair(hash, wtxIn)); - CWalletTx& wtx = (*ret.first).second; - wtx.BindWallet(this); - bool fInsertedNew = ret.second; - if (fInsertedNew) - { - wtx.nTimeReceived = GetAdjustedTime(); - wtx.nOrderPos = IncOrderPosNext(pwalletdb); - wtxOrdered.insert(make_pair(wtx.nOrderPos, TxPair(&wtx, (CAccountingEntry*)0))); - - wtx.nTimeSmart = wtx.nTimeReceived; - if (!wtxIn.hashUnset()) + + wtx.nTimeSmart = wtx.nTimeReceived; + if (!wtxIn.hashUnset()) + { + if (mapBlockIndex.count(wtxIn.hashBlock)) { - if (mapBlockIndex.count(wtxIn.hashBlock)) + int64_t latestNow = wtx.nTimeReceived; + int64_t latestEntry = 0; { - int64_t latestNow = wtx.nTimeReceived; - int64_t latestEntry = 0; + // Tolerate times up to the last timestamp in the wallet not more than 5 minutes into the future + int64_t latestTolerated = latestNow + 300; + const TxItems & txOrdered = wtxOrdered; + for (TxItems::const_reverse_iterator it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) { - // Tolerate times up to the last timestamp in the wallet not more than 5 minutes into the future - int64_t latestTolerated = latestNow + 300; - const TxItems & txOrdered = wtxOrdered; - for (TxItems::const_reverse_iterator it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) + CWalletTx *const pwtx = (*it).second.first; + if (pwtx == &wtx) + continue; + CAccountingEntry *const pacentry = (*it).second.second; + int64_t nSmartTime; + if (pwtx) { - CWalletTx *const pwtx = (*it).second.first; - if (pwtx == &wtx) - continue; - CAccountingEntry *const pacentry = (*it).second.second; - int64_t nSmartTime; - if (pwtx) - { - nSmartTime = pwtx->nTimeSmart; - if (!nSmartTime) - nSmartTime = pwtx->nTimeReceived; - } - else - nSmartTime = pacentry->nTime; - if (nSmartTime <= latestTolerated) - { - latestEntry = nSmartTime; - if (nSmartTime > latestNow) - latestNow = nSmartTime; - break; - } + nSmartTime = pwtx->nTimeSmart; + if (!nSmartTime) + nSmartTime = pwtx->nTimeReceived; + } + else + nSmartTime = pacentry->nTime; + if (nSmartTime <= latestTolerated) + { + latestEntry = nSmartTime; + if (nSmartTime > latestNow) + latestNow = nSmartTime; + break; } } - - int64_t blocktime = mapBlockIndex[wtxIn.hashBlock]->GetBlockTime(); - wtx.nTimeSmart = std::max(latestEntry, std::min(blocktime, latestNow)); } - else - LogPrintf("AddToWallet(): found %s in block %s not in index\n", - wtxIn.GetHash().ToString(), - wtxIn.hashBlock.ToString()); + + int64_t blocktime = mapBlockIndex[wtxIn.hashBlock]->GetBlockTime(); + wtx.nTimeSmart = std::max(latestEntry, std::min(blocktime, latestNow)); } - AddToSpends(hash); + else + LogPrintf("AddToWallet(): found %s in block %s not in index\n", + wtxIn.GetHash().ToString(), + wtxIn.hashBlock.ToString()); } + AddToSpends(hash); + } - bool fUpdated = false; - if (!fInsertedNew) + bool fUpdated = false; + if (!fInsertedNew) + { + // Merge + if (!wtxIn.hashUnset() && wtxIn.hashBlock != wtx.hashBlock) { - // Merge - if (!wtxIn.hashUnset() && wtxIn.hashBlock != wtx.hashBlock) - { - wtx.hashBlock = wtxIn.hashBlock; - fUpdated = true; - } - // If no longer abandoned, update - if (wtxIn.hashBlock.IsNull() && wtx.isAbandoned()) - { - wtx.hashBlock = wtxIn.hashBlock; - fUpdated = true; - } - if (wtxIn.nIndex != -1 && (wtxIn.nIndex != wtx.nIndex)) - { - wtx.nIndex = wtxIn.nIndex; - fUpdated = true; - } - if (wtxIn.fFromMe && wtxIn.fFromMe != wtx.fFromMe) - { - wtx.fFromMe = wtxIn.fFromMe; - fUpdated = true; - } + wtx.hashBlock = wtxIn.hashBlock; + fUpdated = true; + } + // If no longer abandoned, update + if (wtxIn.hashBlock.IsNull() && wtx.isAbandoned()) + { + wtx.hashBlock = wtxIn.hashBlock; + fUpdated = true; + } + if (wtxIn.nIndex != -1 && (wtxIn.nIndex != wtx.nIndex)) + { + wtx.nIndex = wtxIn.nIndex; + fUpdated = true; } + if (wtxIn.fFromMe && wtxIn.fFromMe != wtx.fFromMe) + { + wtx.fFromMe = wtxIn.fFromMe; + fUpdated = true; + } + } - //// debug print - LogPrintf("AddToWallet %s %s%s\n", wtxIn.GetHash().ToString(), (fInsertedNew ? "new" : ""), (fUpdated ? "update" : "")); + //// debug print + LogPrintf("AddToWallet %s %s%s\n", wtxIn.GetHash().ToString(), (fInsertedNew ? "new" : ""), (fUpdated ? "update" : "")); - // Write to disk - if (fInsertedNew || fUpdated) - if (!pwalletdb->WriteTx(wtx)) - return false; + // Write to disk + if (fInsertedNew || fUpdated) + if (!walletdb.WriteTx(wtx)) + return false; - // Break debit/credit balance caches: - wtx.MarkDirty(); + // Break debit/credit balance caches: + wtx.MarkDirty(); - // Notify UI of new or updated transaction - NotifyTransactionChanged(this, hash, fInsertedNew ? CT_NEW : CT_UPDATED); + // Notify UI of new or updated transaction + NotifyTransactionChanged(this, hash, fInsertedNew ? CT_NEW : CT_UPDATED); - // notify an external script when a wallet transaction comes in or is updated - std::string strCmd = GetArg("-walletnotify", ""); + // notify an external script when a wallet transaction comes in or is updated + std::string strCmd = GetArg("-walletnotify", ""); - if ( !strCmd.empty()) - { - boost::replace_all(strCmd, "%s", wtxIn.GetHash().GetHex()); - boost::thread t(runCommand, strCmd); // thread runs free - } + if ( !strCmd.empty()) + { + boost::replace_all(strCmd, "%s", wtxIn.GetHash().GetHex()); + boost::thread t(runCommand, strCmd); // thread runs free + } + + return true; +} +bool CWallet::LoadToWallet(const CWalletTx& wtxIn) +{ + uint256 hash = wtxIn.GetHash(); + + mapWallet[hash] = wtxIn; + CWalletTx& wtx = mapWallet[hash]; + wtx.BindWallet(this); + wtxOrdered.insert(make_pair(wtx.nOrderPos, TxPair(&wtx, (CAccountingEntry*)0))); + AddToSpends(hash); + BOOST_FOREACH(const CTxIn& txin, wtx.vin) { + if (mapWallet.count(txin.prevout.hash)) { + CWalletTx& prevtx = mapWallet[txin.prevout.hash]; + if (prevtx.nIndex == -1 && !prevtx.hashUnset()) { + MarkConflicted(prevtx.hashBlock, wtx.GetHash()); + } + } } + return true; } @@ -900,11 +914,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlock* pbl if (pblock) wtx.SetMerkleBranch(*pblock); - // Do not flush the wallet here for performance reasons - // this is safe, as in case of a crash, we rescan the necessary blocks on startup through our SetBestChain-mechanism - CWalletDB walletdb(strWalletFile, "r+", false); - - return AddToWallet(wtx, false, &walletdb); + return AddToWallet(wtx, false); } } return false; @@ -1166,16 +1176,42 @@ CAmount CWallet::GetChange(const CTransaction& tx) const return nChange; } -bool CWallet::SetHDMasterKey(const CKey& key) +CPubKey CWallet::GenerateNewHDMasterKey() { - LOCK(cs_wallet); + CKey key; + key.MakeNewKey(true); + + int64_t nCreationTime = GetTime(); + CKeyMetadata metadata(nCreationTime); - // store the key as normal "key"/"ckey" object - // in the database - // key metadata is not required + // calculate the pubkey CPubKey pubkey = key.GetPubKey(); - if (!AddKeyPubKey(key, pubkey)) - throw std::runtime_error("CWallet::GenerateNewKey(): AddKey failed"); + assert(key.VerifyPubKey(pubkey)); + + // set the hd keypath to "m" -> Master, refers the masterkeyid to itself + metadata.hdKeypath = "m"; + metadata.hdMasterKeyID = pubkey.GetID(); + + { + LOCK(cs_wallet); + + // mem store the metadata + mapKeyMetadata[pubkey.GetID()] = metadata; + + // write the key&metadata to the database + if (!AddKeyPubKey(key, pubkey)) + throw std::runtime_error("CWallet::GenerateNewKey(): AddKey failed"); + } + + return pubkey; +} + +bool CWallet::SetHDMasterKey(const CPubKey& pubkey) +{ + LOCK(cs_wallet); + + // ensure this wallet.dat can only be opened by clients supporting HD + SetMinVersion(FEATURE_HD); // store the keyid (hash160) together with // the child index counter in the database @@ -2356,7 +2392,7 @@ bool CWallet::CreateTransaction(const vector<CRecipient>& vecSend, CWalletTx& wt *static_cast<CTransaction*>(&wtxNew) = CTransaction(txNew); // Limit size - if (GetTransactionCost(txNew) >= MAX_STANDARD_TX_COST) + if (GetTransactionWeight(txNew) >= MAX_STANDARD_TX_WEIGHT) { strFailReason = _("Transaction too large"); return false; @@ -2411,17 +2447,12 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey) LOCK2(cs_main, cs_wallet); LogPrintf("CommitTransaction:\n%s", wtxNew.ToString()); { - // This is only to keep the database open to defeat the auto-flush for the - // duration of this scope. This is the only place where this optimization - // maybe makes sense; please don't do it anywhere else. - CWalletDB* pwalletdb = fFileBacked ? new CWalletDB(strWalletFile,"r+") : NULL; - // Take key pair from key pool so it won't be used again reservekey.KeepKey(); // Add tx to wallet, because if it has change it's also ours, // otherwise just for transaction history. - AddToWallet(wtxNew, false, pwalletdb); + AddToWallet(wtxNew); // Notify that old coins are spent set<CWalletTx*> setCoins; @@ -2431,9 +2462,6 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey) coin.BindWallet(this); NotifyTransactionChanged(this, coin.GetHash(), CT_UPDATED); } - - if (fFileBacked) - delete pwalletdb; } // Track how many getdata requests our transaction gets @@ -3296,8 +3324,8 @@ bool CWallet::InitLoadWallet() if (GetBoolArg("-usehd", DEFAULT_USE_HD_WALLET) && walletInstance->hdChain.masterKeyID.IsNull()) { // generate a new master key CKey key; - key.MakeNewKey(true); - if (!walletInstance->SetHDMasterKey(key)) + CPubKey masterPubKey = walletInstance->GenerateNewHDMasterKey(); + if (!walletInstance->SetHDMasterKey(masterPubKey)) throw std::runtime_error("CWallet::GenerateNewKey(): Storing master key failed"); } CPubKey newDefaultKey; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index e9d669a7d1..50c94ccfbc 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -78,7 +78,8 @@ enum WalletFeature FEATURE_WALLETCRYPT = 40000, // wallet encryption FEATURE_COMPRPUBKEY = 60000, // compressed public keys - FEATURE_LATEST = 60000 + FEATURE_HD = 130000, // Hierarchical key derivation after BIP32 (HD Wallet) + FEATURE_LATEST = FEATURE_COMPRPUBKEY // HD is optional, use FEATURE_COMPRPUBKEY as latest version }; @@ -580,6 +581,7 @@ private: /* the HD chain data model (external chain counters) */ CHDChain hdChain; + bool fFileBacked; public: /* * Main wallet lock. @@ -590,7 +592,6 @@ public: */ mutable CCriticalSection cs_wallet; - bool fFileBacked; std::string strWalletFile; std::set<int64_t> setKeyPool; @@ -728,7 +729,8 @@ public: bool GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bForceNew = false); void MarkDirty(); - bool AddToWallet(const CWalletTx& wtxIn, bool fFromLoadWallet, CWalletDB* pwalletdb); + bool AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose=true); + bool LoadToWallet(const CWalletTx& wtxIn); void SyncTransaction(const CTransaction& tx, const CBlockIndex *pindex, const CBlock* pblock); bool AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlock* pblock, bool fUpdate); int ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false); @@ -898,10 +900,13 @@ public: /* Set the HD chain model (chain child index counters) */ bool SetHDChain(const CHDChain& chain, bool memonly); + const CHDChain& GetHDChain() { return hdChain; } + /* Generates a new HD master key (will not be activated) */ + CPubKey GenerateNewHDMasterKey(); + /* Set the current HD master key (will reset the chain child index counters) */ - bool SetHDMasterKey(const CKey& key); - const CHDChain& GetHDChain() { return hdChain; } + bool SetHDMasterKey(const CPubKey& key); }; /** A key allocated from the key pool. */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 72af8ab7b2..543522ca64 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -400,7 +400,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, if (wtx.nOrderPos == -1) wss.fAnyUnordered = true; - pwallet->AddToWallet(wtx, true, NULL); + pwallet->LoadToWallet(wtx); } else if (strType == "acentry") { |