diff options
36 files changed, 899 insertions, 265 deletions
diff --git a/contrib/init/bitcoind.openrc b/contrib/init/bitcoind.openrc index 50377c0995..86222295db 100644 --- a/contrib/init/bitcoind.openrc +++ b/contrib/init/bitcoind.openrc @@ -1,4 +1,4 @@ -#!/sbin/runscript +#!/sbin/openrc-run # backward compatibility for existing gentoo layout # diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp new file mode 100644 index 0000000000..0f6d993fd2 --- /dev/null +++ b/contrib/valgrind.supp @@ -0,0 +1,43 @@ +# Valgrind suppressions file for Bitcoin. +# +# Includes known Valgrind warnings in our dependencies that cannot be fixed +# in-tree. +# +# Example use: +# $ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin +# $ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ +# --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite +{ + Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 + Memcheck:Leak + match-leak-kinds: reachable + fun:malloc + obj:*/libstdc++.* + fun:call_init.part.0 + fun:call_init + fun:_dl_init + obj:*/ld-*.so +} +{ + Suppress libdb warning - https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=662917 + Memcheck:Cond + obj:*/libdb_cxx-*.so + fun:__log_put + obj:*/libdb_cxx-*.so + fun:__log_put_record +} +{ + Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7leveldbL10InitModuleEv +} +{ + Suppress leveldb warning (leveldb::Env::Default()) - https://github.com/google/leveldb/issues/113 + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + ... + fun:_ZN7leveldbL14InitDefaultEnvEv +} diff --git a/doc/build-windows.md b/doc/build-windows.md index 0d96af26a2..9e3e38d188 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -4,11 +4,14 @@ WINDOWS BUILD NOTES Below are some notes on how to build Bitcoin Core for Windows. Most developers use cross-compilation from Ubuntu to build executables for -Windows. Cross-compilation is also used to build the release binaries. +Windows. This is also used to build the release binaries. -Currently only building on Ubuntu Trusty 14.04 or Ubuntu Zesty 17.04 or later is supported. -Building on Ubuntu Xenial 16.04 is known to be broken, see extensive discussion in issue [8732](https://github.com/bitcoin/bitcoin/issues/8732). -While it may be possible to do so with work arounds, it's potentially dangerous and not recommended. +Building on Ubuntu Trusty 14.04 is recommended. +At the time of writing the Windows Subsystem for Linux installs Ubuntu Xenial 16.04. The default cross +compiler package for Ubuntu Xenial does not produce working executables for some of the bitcoin applications. +It is possible to build on Ubuntu Xenial by installing the cross compiler packages from +Ubuntu Zesty, see the steps below. +Building on Ubuntu Zesty 17.04 up to 17.10 has been verified to work. While there are potentially a number of ways to build on Windows (for example using msys / mingw-w64), using the Windows Subsystem For Linux is the most straightforward. If you are building with @@ -19,7 +22,7 @@ Compiling with Windows Subsystem For Linux ------------------------------------------- With Windows 10, Microsoft has released a new feature named the [Windows -Subsystem for Linux](https://msdn.microsoft.com/commandline/wsl/about). This +Subsystem for Linux (WSL)](https://msdn.microsoft.com/commandline/wsl/about). This feature allows you to run a bash shell directly on Windows in an Ubuntu-based environment. Within this environment you can cross compile for Windows without the need for a separate Linux VM or server. @@ -28,20 +31,19 @@ This feature is not supported in versions of Windows prior to Windows 10 or on Windows Server SKUs. In addition, it is available [only for 64-bit versions of Windows](https://msdn.microsoft.com/en-us/commandline/wsl/install_guide). -To get the bash shell, you must first activate the feature in Windows. +Full instructions to install WSL are available on the above link. +To install WSL on Windows 10 with Fall Creators Update installed (version >= 16215.0) do the following: -1. Turn on Developer Mode - * Open Settings -> Update and Security -> For developers - * Select the Developer Mode radio button - * Restart if necessary -2. Enable the Windows Subsystem for Linux feature +1. Enable the Windows Subsystem for Linux feature * From Start, search for "Turn Windows features on or off" (type 'turn') - * Select Windows Subsystem for Linux (beta) + * Select Windows Subsystem for Linux * Click OK * Restart if necessary +2. Install Ubuntu + * Open Microsoft Store and search for Ubuntu or use [this link](https://www.microsoft.com/store/productId/9NBLGGH4MSV6) + * Click Install 3. Complete Installation - * Open a cmd prompt and type "bash" - * Accept the license + * Open a cmd prompt and type "Ubuntu" * Create a new UNIX user account (this is a separate account from your Windows account) After the bash shell is active, you can follow the instructions below, starting @@ -57,7 +59,7 @@ installing the toolchain will be different. First, install the general dependencies: - sudo apt-get install build-essential libtool autotools-dev automake pkg-config bsdmainutils curl + sudo apt install build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git A host toolchain (`build-essential`) is necessary because some dependency packages (such as `protobuf`) need to build host utilities that are used in the @@ -65,20 +67,32 @@ build process. See also: [dependencies.md](dependencies.md). -If you're building on Ubuntu 17.04 or later, run these two commands, selecting the 'posix' variant for both, -to work around issues with mingw-w64. See issue [8732](https://github.com/bitcoin/bitcoin/issues/8732) for more information. -``` -sudo update-alternatives --config x86_64-w64-mingw32-g++ -sudo update-alternatives --config x86_64-w64-mingw32-gcc -``` - ## Building for 64-bit Windows -To build executables for Windows 64-bit, install the following dependencies: +The first step is to install the mingw-w64 cross-compilation tool chain. Due to different Ubuntu +packages for each distribution and problems with the Xenial packages the steps for each are different. - sudo apt-get install g++-mingw-w64-x86-64 mingw-w64-x86-64-dev +Common steps to install mingw32 cross compiler tool chain: -Then build using: + sudo apt install g++-mingw-w64-x86-64 + +Ubuntu Trusty 14.04: + + No further steps required + +Ubuntu Xenial 16.04 and Windows Subsystem for Linux <sup>[1](#footnote1),[2](#footnote2)</sup>: + + sudo apt install software-properties-common + sudo add-apt-repository "deb http://archive.ubuntu.com/ubuntu zesty universe" + sudo apt update + sudo apt upgrade + sudo update-alternatives --config x86_64-w64-mingw32-g++ # Set the default mingw32 g++ compiler option to posix. + +Ubuntu Zesty 17.04 <sup>[2](#footnote2)</sup>: + + sudo update-alternatives --config x86_64-w64-mingw32-g++ # Set the default mingw32 g++ compiler option to posix. + +Once the tool chain is installed the build steps are common: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var cd depends @@ -92,7 +106,11 @@ Then build using: To build executables for Windows 32-bit, install the following dependencies: - sudo apt-get install g++-mingw-w64-i686 mingw-w64-i686-dev + sudo apt install g++-mingw-w64-i686 mingw-w64-i686-dev + +For Ubuntu Xenial 16.04, Ubuntu Zesty 17.04 and Windows Subsystem for Linux <sup>[2](#footnote2)</sup>: + + sudo update-alternatives --config i686-w64-mingw32-g++ # Set the default mingw32 g++ compiler option to posix. Then build using: @@ -117,3 +135,20 @@ as they appear in the release `.zip` archive. This can be done in the following way. This will install to `c:\workspace\bitcoin`, for example: make install DESTDIR=/mnt/c/workspace/bitcoin + +Footnotes +--------- + +<a name="footnote1">1</a>: There is currently a bug in the 64 bit mingw-w64 cross compiler packaged for WSL/Ubuntu Xenial 16.04 that +causes two of the bitcoin executables to crash shortly after start up. The bug is related to the +-fstack-protector-all g++ compiler flag which is used to mitigate buffer overflows. +Installing the mingw-w64 packages from the Ubuntu 17 distribution solves the issue, however, this is not +an officially supported approach and it's only recommended if you are prepared to reinstall WSL/Ubuntu should +something break. + +<a name="footnote2">2</a>: Starting from Ubuntu Xenial 16.04 both the 32 and 64 bit mingw-w64 packages install two different +compiler options to allow a choice between either posix or win32 threads. The default option is win32 threads which is the more +efficient since it will result in binary code that links directly with the Windows kernel32.lib. Unfortunately, the headers +required to support win32 threads conflict with some of the classes in the C++11 standard library in particular std::mutex. +It's not possible to build the bitcoin code using the win32 version of the mingw-w64 cross compilers (at least not without +modifying headers in the bitcoin source code). diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 33c6ab9cb3..35797da7ae 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -167,6 +167,37 @@ can be very difficult to track down. Compiling with -DDEBUG_LOCKORDER (configure CXXFLAGS="-DDEBUG_LOCKORDER -g") inserts run-time checks to keep track of which locks are held, and adds warnings to the debug.log file if inconsistencies are detected. +**Valgrind suppressions file** + +Valgrind is a programming tool for memory debugging, memory leak detection, and +profiling. The repo contains a Valgrind suppressions file +([`valgrind.supp`](https://github.com/bitcoin/bitcoin/blob/master/contrib/valgrind.supp)) +which includes known Valgrind warnings in our dependencies that cannot be fixed +in-tree. Example use: + +```shell +$ valgrind --suppressions=contrib/valgrind.supp src/test/test_bitcoin +$ valgrind --suppressions=contrib/valgrind.supp --leak-check=full \ + --show-leak-kinds=all src/test/test_bitcoin --log_level=test_suite +$ valgrind -v --leak-check=full src/bitcoind -printtoconsole +``` + +**compiling for test coverage** + +LCOV can be used to generate a test coverage report based upon `make check` +execution. LCOV must be installed on your system (e.g. the `lcov` package +on Debian/Ubuntu). + +To enable LCOV report generation during test runs: + +```shell +./configure --enable-lcov +make +make cov + +# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`. +``` + Locking/mutex usage notes ------------------------- @@ -644,3 +675,16 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: If a RPC response is not a JSON object then it is harder to avoid API breakage if new data in the response is needed. + +- Wallet RPCs call BlockUntilSyncedToCurrentChain to maintain consistency with + `getblockchaininfo`'s state immediately prior to the call's execution. Wallet + RPCs whose behavior does *not* depend on the current chainstate may omit this + call. + + - *Rationale*: In previous versions of Bitcoin Core, the wallet was always + in-sync with the chainstate (by virtue of them all being updated in the + same cs_main lock). In order to maintain the behavior that wallet RPCs + return results as of at least the highest best-known block an RPC + client may be aware of prior to entering a wallet RPC call, we must block + until the wallet is caught up to the chainstate as of the RPC call's entry. + This also makes the API much easier for RPC clients to reason about. diff --git a/doc/release-notes.md b/doc/release-notes.md index 23414666ce..6e4fed0a2d 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -56,18 +56,6 @@ frequently tested on them. Notable changes =============== -Miner block size limiting deprecated ------------------------------------- - -Though blockmaxweight has been preferred for limiting the size of blocks returned by -getblocktemplate since 0.13.0, blockmaxsize remained as an option for those who wished -to limit their block size directly. Using this option resulted in a few UI issues as -well as non-optimal fee selection and ever-so-slightly worse performance, and has thus -now been deprecated. Further, the blockmaxsize option is now used only to calculate an -implied blockmaxweight, instead of limiting block size directly. Any miners who wish -to limit their blocks by size, instead of by weight, will have to do so manually by -removing transactions from their block template directly. - HD-wallets by default --------------------- Due to a backward-incompatible change in the wallet database, wallets created @@ -76,18 +64,13 @@ will only create hierarchical deterministic (HD) wallets. Low-level RPC changes ---------------------- -- `listsinceblock` will now throw an error if an unknown `blockhash` argument - value is passed, instead of returning a list of all wallet transactions since - the genesis block. -- The "currentblocksize" value in getmininginfo has been removed. - The deprecated RPC `getinfo` was removed. It is recommended that the more specific RPCs are used: * `getblockchaininfo` * `getnetworkinfo` * `getwalletinfo` * `getmininginfo` +- The wallet RPC `getreceivedbyaddress` will return an error if called with an address not in the wallet. -- `dumpwallet` no longer allows overwriting files. This is a security measure - as well as prevents dangerous user mistakes. Credits ======= diff --git a/doc/release-notes/release-notes-0.15.1.md b/doc/release-notes/release-notes-0.15.1.md new file mode 100644 index 0000000000..75d2e09714 --- /dev/null +++ b/doc/release-notes/release-notes-0.15.1.md @@ -0,0 +1,277 @@ +Bitcoin Core version *0.15.1* is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.15.1/> + +or + + <https://bitcoin.org/bin/bitcoin-core-0.15.1/> + +This is a new minor version release, including various bugfixes and +performance improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes for older versions), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +The first time you run version 0.15.0 or higher, your chainstate database will +be converted to a new format, which will take anywhere from a few minutes to +half an hour, depending on the speed of your machine. + +The file format of `fee_estimates.dat` changed in version 0.15.0. Hence, a +downgrade from version 0.15 or upgrade to version 0.15 will cause all fee +estimates to be discarded. + +Note that the block database format also changed in version 0.8.0 and there is no +automatic upgrade code from before version 0.8 to version 0.15.0. Upgrading +directly from 0.7.x and earlier without redownloading the blockchain is not supported. +However, as usual, old wallet versions are still supported. + +Downgrading warning +------------------- + +The chainstate database for this release is not compatible with previous +releases, so if you run 0.15 and then decide to switch back to any +older version, you will need to run the old release with the `-reindex-chainstate` +option to rebuild the chainstate data structures in the old format. + +If your node has pruning enabled, this will entail re-downloading and +processing the entire blockchain. + +Compatibility +============== + +Bitcoin Core is extensively tested on multiple operating systems using +the Linux kernel, macOS 10.8+, and Windows Vista and later. Windows XP is not supported. + +Bitcoin Core should also work on most other Unix-like systems but is not +frequently tested on them. + + +Notable changes +=============== + +Network fork safety enhancements +-------------------------------- + +A number of changes to the way Bitcoin Core deals with peer connections and invalid blocks +have been made, as a safety precaution against blockchain forks and misbehaving peers. + +- Unrequested blocks with less work than the minimum-chain-work are now no longer processed even +if they have more work than the tip (a potential issue during IBD where the tip may have low-work). +This prevents peers wasting the resources of a node. + +- Peers which provide a chain with less work than the minimum-chain-work during IBD will now be disconnected. + +- For a given outbound peer, we now check whether their best known block has at least as much work as our tip. If it +doesn't, and if we still haven't heard about a block with sufficient work after a 20 minute timeout, then we send +a single getheaders message, and wait 2 more minutes. If after two minutes their best known block has insufficient +work, we disconnect that peer. We protect 4 of our outbound peers from being disconnected by this logic to prevent +excessive network topology changes as a result of this algorithm, while still ensuring that we have a reasonable +number of nodes not known to be on bogus chains. + +- Outbound (non-manual) peers that serve us block headers that are already known to be invalid (other than compact +block announcements, because BIP 152 explicitly permits nodes to relay compact blocks before fully validating them) +will now be disconnected. + +- If the chain tip has not been advanced for over 30 minutes, we now assume the tip may be stale and will try to connect +to an additional outbound peer. A periodic check ensures that if this extra peer connection is in use, we will disconnect +the peer that least recently announced a new block. + +- The set of all known invalid-themselves blocks (i.e. blocks which we attempted to connect but which were found to be +invalid) are now tracked and used to check if new headers build on an invalid chain. This ensures that everything that +descends from an invalid block is marked as such. + + +Miner block size limiting deprecated +------------------------------------ + +Though blockmaxweight has been preferred for limiting the size of blocks returned by +getblocktemplate since 0.13.0, blockmaxsize remained as an option for those who wished +to limit their block size directly. Using this option resulted in a few UI issues as +well as non-optimal fee selection and ever-so-slightly worse performance, and has thus +now been deprecated. Further, the blockmaxsize option is now used only to calculate an +implied blockmaxweight, instead of limiting block size directly. Any miners who wish +to limit their blocks by size, instead of by weight, will have to do so manually by +removing transactions from their block template directly. + + +GUI settings backed up on reset +------------------------------- + +The GUI settings will now be written to `guisettings.ini.bak` in the data directory before wiping them when +the `-resetguisettings` argument is used. This can be used to retroactively troubleshoot issues due to the +GUI settings. + + +Duplicate wallets disallowed +---------------------------- + +Previously, it was possible to open the same wallet twice by manually copying the wallet file, causing +issues when both were opened simultaneously. It is no longer possible to open copies of the same wallet. + + +Debug `-minimumchainwork` argument added +---------------------------------------- + +A hidden debug argument `-minimumchainwork` has been added to allow a custom minimum work value to be used +when validating a chain. + + +Low-level RPC changes +---------------------- + +- The "currentblocksize" value in getmininginfo has been removed. + +- `dumpwallet` no longer allows overwriting files. This is a security measure + as well as prevents dangerous user mistakes. + +- `backupwallet` will now fail when attempting to backup to source file, rather than + destroying the wallet. + +- `listsinceblock` will now throw an error if an unknown `blockhash` argument + value is passed, instead of returning a list of all wallet transactions since + the genesis block. The behaviour is unchanged when an empty string is provided. + +0.15.1 Change log +================= + +### Mining +- #11100 `7871a7d` Fix confusing blockmax{size,weight} options, dont default to throwing away money (TheBlueMatt) + +### RPC and other APIs +- #10859 `2a5d099` gettxout: Slightly improve doc and tests (jtimon) +- #11267 `b1a6c94` update cli for estimate\*fee argument rename (laanwj) +- #11483 `20cdc2b` Fix importmulti bug when importing an already imported key (pedrobranco) +- #9937 `a43be5b` Prevent `dumpwallet` from overwriting files (laanwj) +- #11465 `405e069` Update named args documentation for importprivkey (dusty-wil) +- #11131 `b278a43` Write authcookie atomically (laanwj) +- #11565 `7d4546f` Make listsinceblock refuse unknown block hash (ryanofsky) +- #11593 `8195cb0` Work-around an upstream libevent bug (theuni) + +### P2P protocol and network code +- #11397 `27e861a` Improve and document SOCKS code (laanwj) +- #11252 `0fe2a9a` When clearing addrman clear mapInfo and mapAddr (instagibbs) +- #11527 `a2bd86a` Remove my testnet DNS seed (schildbach) +- #10756 `0a5477c` net processing: swap out signals for an interface class (theuni) +- #11531 `55b7abf` Check that new headers are not a descendant of an invalid block (more effeciently) (TheBlueMatt) +- #11560 `49bf090` Connect to a new outbound peer if our tip is stale (sdaftuar) +- #11568 `fc966bb` Disconnect outbound peers on invalid chains (sdaftuar) +- #11578 `ec8dedf` Add missing lock in ProcessHeadersMessage(...) (practicalswift) +- #11456 `6f27965` Replace relevant services logic with a function suite (TheBlueMatt) +- #11490 `bf191a7` Disconnect from outbound peers with bad headers chains (sdaftuar) + +### Validation +- #10357 `da4908c` Allow setting nMinimumChainWork on command line (sdaftuar) +- #11458 `2df65ee` Don't process unrequested, low-work blocks (sdaftuar) + +### Build system +- #11440 `b6c0209` Fix validationinterface build on super old boost/clang (TheBlueMatt) +- #11530 `265bb21` Add share/rpcuser to dist. source code archive (MarcoFalke) + +### GUI +- #11334 `19d63e8` Remove custom fee radio group and remove nCustomFeeRadio setting (achow101) +- #11198 `7310f1f` Fix display of package name on 'open config file' tooltip (esotericnonsense) +- #11015 `6642558` Add delay before filtering transactions (lclc) +- #11338 `6a62c74` Backup former GUI settings on `-resetguisettings` (laanwj) +- #11335 `8d13b42` Replace save|restoreWindowGeometry with Qt functions (MeshCollider) +- #11237 `2e31b1d` Fixing division by zero in time remaining (MeshCollider) +- #11247 `47c02a8` Use IsMine to validate custom change address (MarcoFalke) + +### Wallet +- #11017 `9e8aae3` Close DB on error (kallewoof) +- #11225 `6b4d9f2` Update stored witness in AddToWallet (sdaftuar) +- #11126 `2cb720a` Acquire cs_main lock before cs_wallet during wallet initialization (ryanofsky) +- #11476 `9c8006d` Avoid opening copied wallet databases simultaneously (ryanofsky) +- #11492 `de7053f` Fix leak in CDB constructor (promag) +- #11376 `fd79ed6` Ensure backupwallet fails when attempting to backup to source file (tomasvdw) +- #11326 `d570aa4` Fix crash on shutdown with invalid wallet (MeshCollider) + +### Tests and QA +- #11399 `a825d4a` Fix bip68-sequence rpc test (jl2012) +- #11150 `847c75e` Add getmininginfo test (mess110) +- #11407 `806c78f` add functional test for mempoolreplacement command line arg (instagibbs) +- #11433 `e169349` Restore bitcoin-util-test py2 compatibility (MarcoFalke) +- #11308 `2e1ac70` zapwallettxes: Wait up to 3s for mempool reload (MarcoFalke) +- #10798 `716066d` test bitcoin-cli (jnewbery) +- #11443 `019c492` Allow "make cov" out-of-tree; Fix rpc mapping check (MarcoFalke) +- #11445 `51bad91` 0.15.1 Backports (MarcoFalke) +- #11319 `2f0b30a` Fix error introduced into p2p-segwit.py, and prevent future similar errors (sdaftuar) +- #10552 `e4605d9` Tests for zmqpubrawtx and zmqpubrawblock (achow101) +- #11067 `eeb24a3` TestNode: Add wait_until_stopped helper method (MarcoFalke) +- #11068 `5398f20` Move wait_until to util (MarcoFalke) +- #11125 `812c870` Add bitcoin-cli -stdin and -stdinrpcpass functional tests (promag) +- #11077 `1d80d1e` fix timeout issues from TestNode (jnewbery) +- #11078 `f1ced0d` Make p2p-leaktests.py more robust (jnewbery) +- #11210 `f3f7891` Stop test_bitcoin-qt touching ~/.bitcoin (MeshCollider) +- #11234 `f0b6795` Remove redundant testutil.cpp|h files (MeshCollider) +- #11215 `cef0319` fixups from set_test_params() (jnewbery) +- #11345 `f9cf7b5` Check connectivity before sending in assumevalid.py (jnewbery) +- #11091 `c276c1e` Increase initial RPC timeout to 60 seconds (laanwj) +- #10711 `fc2aa09` Introduce TestNode (jnewbery) +- #11230 `d8dd8e7` Fixup dbcrash interaction with add_nodes() (jnewbery) +- #11241 `4424176` Improve signmessages functional test (mess110) +- #11116 `2c4ff35` Unit tests for script/standard and IsMine functions (jimpo) +- #11422 `a36f332` Verify DBWrapper iterators are taking snapshots (TheBlueMatt) +- #11121 `bb5e7cb` TestNode tidyups (jnewbery) +- #11521 `ca0f3f7` travis: move back to the minimal image (theuni) +- #11538 `adbc9d1` Fix race condition failures in replace-by-fee.py, sendheaders.py (sdaftuar) +- #11472 `4108879` Make tmpdir option an absolute path, misc cleanup (MarcoFalke) +- #10853 `5b728c8` Fix RPC failure testing (again) (jnewbery) +- #11310 `b6468d3` Test listwallets RPC (mess110) + +### Miscellaneous +- #11377 `75997c3` Disallow uncompressed pubkeys in bitcoin-tx [multisig] output adds (TheBlueMatt) +- #11437 `dea3b87` [Docs] Update Windows build instructions for using WSL and Ubuntu 17.04 (fanquake) +- #11318 `8b61aee` Put back inadvertently removed copyright notices (gmaxwell) +- #11442 `cf18f42` [Docs] Update OpenBSD Build Instructions for OpenBSD 6.2 (fanquake) +- #10957 `50bd3f6` Avoid returning a BIP9Stats object with uninitialized values (practicalswift) +- #11539 `01223a0` [verify-commits] Allow revoked keys to expire (TheBlueMatt) + + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andreas Schildbach +- Andrew Chow +- Chris Moore +- Cory Fields +- Cristian Mircea Messel +- Daniel Edgecumbe +- Donal OConnor +- Dusty Williams +- fanquake +- Gregory Sanders +- Jim Posen +- John Newbery +- Johnson Lau +- João Barbosa +- Jorge Timón +- Karl-Johan Alm +- Lucas Betschart +- MarcoFalke +- Matt Corallo +- Paul Berg +- Pedro Branco +- Pieter Wuille +- practicalswift +- Russell Yanofsky +- Samuel Dobson +- Suhas Daftuar +- Tomas van der Wansem +- Wladimir J. van der Laan + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). diff --git a/src/arith_uint256.h b/src/arith_uint256.h index 5fd4fe96cf..1009fe1cc8 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -25,7 +25,7 @@ template<unsigned int BITS> class base_uint { protected: - enum { WIDTH=BITS/32 }; + static constexpr int WIDTH = BITS / 32; uint32_t pn[WIDTH]; public: diff --git a/src/chain.h b/src/chain.h index f1036e5d92..0667e0121f 100644 --- a/src/chain.h +++ b/src/chain.h @@ -304,7 +304,7 @@ public: return (int64_t)nTimeMax; } - enum { nMedianTimeSpan=11 }; + static constexpr int nMedianTimeSpan = 11; int64_t GetMedianTimePast() const { diff --git a/src/consensus/consensus.h b/src/consensus/consensus.h index ddd4ee9fab..6e3bac2d0e 100644 --- a/src/consensus/consensus.h +++ b/src/consensus/consensus.h @@ -24,12 +24,9 @@ static const size_t MIN_TRANSACTION_WEIGHT = WITNESS_SCALE_FACTOR * 60; // 60 is static const size_t MIN_SERIALIZABLE_TRANSACTION_WEIGHT = WITNESS_SCALE_FACTOR * 10; // 10 is the lower bound for the size of a serialized CTransaction /** Flags for nSequence and nLockTime locks */ -enum { - /* Interpret sequence numbers as relative lock-time constraints. */ - LOCKTIME_VERIFY_SEQUENCE = (1 << 0), - - /* Use GetMedianTimePast() instead of nTime for end point timestamp. */ - LOCKTIME_MEDIAN_TIME_PAST = (1 << 1), -}; +/** Interpret sequence numbers as relative lock-time constraints. */ +static constexpr unsigned int LOCKTIME_VERIFY_SEQUENCE = (1 << 0); +/** Use GetMedianTimePast() instead of nTime for end point timestamp. */ +static constexpr unsigned int LOCKTIME_MEDIAN_TIME_PAST = (1 << 1); #endif // BITCOIN_CONSENSUS_CONSENSUS_H diff --git a/src/init.cpp b/src/init.cpp index 7ac2cf2d32..9bdde00558 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -261,6 +261,7 @@ void Shutdown() #endif UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); + GetMainSignals().UnregisterWithMempoolSignals(mempool); #ifdef ENABLE_WALLET CloseWallets(); #endif @@ -1236,6 +1237,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) threadGroup.create_thread(boost::bind(&TraceThread<CScheduler::Function>, "scheduler", serviceLoop)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + GetMainSignals().RegisterWithMempoolSignals(mempool); /* Start the RPC server already. It will be started in "warmup" mode * and not really process calls already (but it will signify connections diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 8e503f89db..38b9703d91 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3006,6 +3006,7 @@ void PeerLogicValidation::ConsiderEviction(CNode *pto, int64_t time_in_seconds) LogPrintf("Disconnecting outbound peer %d for old chain, best known block = %s\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>"); pto->fDisconnect = true; } else { + assert(state.m_chain_sync.m_work_header); LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto->GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); state.m_chain_sync.m_sent_getheaders = true; diff --git a/src/protocol.h b/src/protocol.h index 56b59aed3f..558616eaf8 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -27,16 +27,13 @@ class CMessageHeader { public: - enum { - MESSAGE_START_SIZE = 4, - COMMAND_SIZE = 12, - MESSAGE_SIZE_SIZE = 4, - CHECKSUM_SIZE = 4, - - MESSAGE_SIZE_OFFSET = MESSAGE_START_SIZE + COMMAND_SIZE, - CHECKSUM_OFFSET = MESSAGE_SIZE_OFFSET + MESSAGE_SIZE_SIZE, - HEADER_SIZE = MESSAGE_START_SIZE + COMMAND_SIZE + MESSAGE_SIZE_SIZE + CHECKSUM_SIZE - }; + static constexpr size_t MESSAGE_START_SIZE = 4; + static constexpr size_t COMMAND_SIZE = 12; + static constexpr size_t MESSAGE_SIZE_SIZE = 4; + static constexpr size_t CHECKSUM_SIZE = 4; + static constexpr size_t MESSAGE_SIZE_OFFSET = MESSAGE_START_SIZE + COMMAND_SIZE; + static constexpr size_t CHECKSUM_OFFSET = MESSAGE_SIZE_OFFSET + MESSAGE_SIZE_SIZE; + static constexpr size_t HEADER_SIZE = MESSAGE_START_SIZE + COMMAND_SIZE + MESSAGE_SIZE_SIZE + CHECKSUM_SIZE; typedef unsigned char MessageStartChars[MESSAGE_START_SIZE]; explicit CMessageHeader(const MessageStartChars& pchMessageStartIn); diff --git a/src/qt/forms/receivecoinsdialog.ui b/src/qt/forms/receivecoinsdialog.ui index 03fcb2fb50..2ca296bc22 100644 --- a/src/qt/forms/receivecoinsdialog.ui +++ b/src/qt/forms/receivecoinsdialog.ui @@ -28,23 +28,6 @@ <layout class="QVBoxLayout" name="verticalLayout_3"> <item> <layout class="QGridLayout" name="gridLayout"> - <item row="7" column="2"> - <widget class="QCheckBox" name="reuseAddress"> - <property name="toolTip"> - <string>Reuse one of the previously used receiving addresses. Reusing addresses has security and privacy issues. Do not use this unless re-generating a payment request made before.</string> - </property> - <property name="text"> - <string>R&euse an existing receiving address (not recommended)</string> - </property> - </widget> - </item> - <item row="7" column="0"> - <widget class="QLabel" name="label_4"> - <property name="text"> - <string/> - </property> - </widget> - </item> <item row="6" column="0"> <widget class="QLabel" name="label_3"> <property name="toolTip"> @@ -127,7 +110,7 @@ </property> </widget> </item> - <item row="8" column="2"> + <item row="7" column="2"> <layout class="QHBoxLayout" name="horizontalLayout"> <item> <widget class="QPushButton" name="receiveButton"> @@ -184,7 +167,7 @@ </item> </layout> </item> - <item row="8" column="0"> + <item row="7" column="0"> <widget class="QLabel" name="label_7"> <property name="text"> <string/> @@ -324,7 +307,6 @@ <tabstop>reqLabel</tabstop> <tabstop>reqAmount</tabstop> <tabstop>reqMessage</tabstop> - <tabstop>reuseAddress</tabstop> <tabstop>receiveButton</tabstop> <tabstop>clearButton</tabstop> <tabstop>recentRequestsView</tabstop> diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 84f43266e1..4e1a9b172f 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -106,7 +106,6 @@ void ReceiveCoinsDialog::clear() ui->reqAmount->clear(); ui->reqLabel->setText(""); ui->reqMessage->setText(""); - ui->reuseAddress->setChecked(false); updateDisplayUnit(); } @@ -135,25 +134,8 @@ void ReceiveCoinsDialog::on_receiveButton_clicked() QString address; QString label = ui->reqLabel->text(); - if(ui->reuseAddress->isChecked()) - { - /* Choose existing receiving address */ - AddressBookPage dlg(platformStyle, AddressBookPage::ForSelection, AddressBookPage::ReceivingTab, this); - dlg.setModel(model->getAddressTableModel()); - if(dlg.exec()) - { - address = dlg.getReturnValue(); - if(label.isEmpty()) /* If no label provided, use the previously used label */ - { - label = model->getAddressTableModel()->labelForAddress(address); - } - } else { - return; - } - } else { - /* Generate new receiving address */ - address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, ""); - } + /* Generate new receiving address */ + address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, ""); SendCoinsRecipient info(address, label, ui->reqAmount->value(), ui->reqMessage->text()); ReceiveRequestDialog *dialog = new ReceiveRequestDialog(this); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 4a3404308d..8fff937d82 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1130,45 +1130,46 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) "Returns an object containing various state info regarding blockchain processing.\n" "\nResult:\n" "{\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" - " \"blocks\": xxxxxx, (numeric) the current number of blocks processed in the server\n" - " \"headers\": xxxxxx, (numeric) the current number of headers we have validated\n" - " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" - " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" - " \"mediantime\": xxxxxx, (numeric) median time for the current best block\n" + " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"blocks\": xxxxxx, (numeric) the current number of blocks processed in the server\n" + " \"headers\": xxxxxx, (numeric) the current number of headers we have validated\n" + " \"bestblockhash\": \"...\", (string) the hash of the currently best block\n" + " \"difficulty\": xxxxxx, (numeric) the current difficulty\n" + " \"mediantime\": xxxxxx, (numeric) median time for the current best block\n" " \"verificationprogress\": xxxx, (numeric) estimate of verification progress [0..1]\n" - " \"chainwork\": \"xxxx\" (string) total amount of work in active chain, in hexadecimal\n" - " \"size_on_disk\": xxxxxx, (numeric) the estimated size of the block and undo files on disk\n" - " \"pruned\": xx, (boolean) if the blocks are subject to pruning\n" - " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored (only present if pruning is enabled)\n" - " \"automatic_pruning\": xx, (boolean) whether automatic pruning is enabled (only present if pruning is enabled)\n" + " \"initialblockdownload\": xxxx, (bool) (debug information) estimate of whether this node is in Initial Block Download mode.\n" + " \"chainwork\": \"xxxx\" (string) total amount of work in active chain, in hexadecimal\n" + " \"size_on_disk\": xxxxxx, (numeric) the estimated size of the block and undo files on disk\n" + " \"pruned\": xx, (boolean) if the blocks are subject to pruning\n" + " \"pruneheight\": xxxxxx, (numeric) lowest-height complete block stored (only present if pruning is enabled)\n" + " \"automatic_pruning\": xx, (boolean) whether automatic pruning is enabled (only present if pruning is enabled)\n" " \"prune_target_size\": xxxxxx, (numeric) the target size used by pruning (only present if automatic pruning is enabled)\n" - " \"softforks\": [ (array) status of softforks in progress\n" + " \"softforks\": [ (array) status of softforks in progress\n" " {\n" - " \"id\": \"xxxx\", (string) name of softfork\n" - " \"version\": xx, (numeric) block version\n" - " \"reject\": { (object) progress toward rejecting pre-softfork blocks\n" - " \"status\": xx, (boolean) true if threshold reached\n" + " \"id\": \"xxxx\", (string) name of softfork\n" + " \"version\": xx, (numeric) block version\n" + " \"reject\": { (object) progress toward rejecting pre-softfork blocks\n" + " \"status\": xx, (boolean) true if threshold reached\n" " },\n" " }, ...\n" " ],\n" - " \"bip9_softforks\": { (object) status of BIP9 softforks in progress\n" - " \"xxxx\" : { (string) name of the softfork\n" - " \"status\": \"xxxx\", (string) one of \"defined\", \"started\", \"locked_in\", \"active\", \"failed\"\n" - " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" - " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" - " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" - " \"since\": xx, (numeric) height of the first block to which the status applies\n" - " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" - " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" - " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" - " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" - " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" - " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" + " \"bip9_softforks\": { (object) status of BIP9 softforks in progress\n" + " \"xxxx\" : { (string) name of the softfork\n" + " \"status\": \"xxxx\", (string) one of \"defined\", \"started\", \"locked_in\", \"active\", \"failed\"\n" + " \"bit\": xx, (numeric) the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)\n" + " \"startTime\": xx, (numeric) the minimum median time past of a block at which the bit gains its meaning\n" + " \"timeout\": xx, (numeric) the median time past of a block at which the deployment is considered failed if not yet locked in\n" + " \"since\": xx, (numeric) height of the first block to which the status applies\n" + " \"statistics\": { (object) numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)\n" + " \"period\": xx, (numeric) the length in blocks of the BIP9 signalling period \n" + " \"threshold\": xx, (numeric) the number of blocks with the version bit set required to activate the feature \n" + " \"elapsed\": xx, (numeric) the number of blocks elapsed since the beginning of the current period \n" + " \"count\": xx, (numeric) the number of blocks with the version bit set in the current period \n" + " \"possible\": xx (boolean) returns false if there are not enough blocks left in this period to pass activation threshold \n" " }\n" " }\n" " }\n" - " \"warnings\" : \"...\", (string) any network and blockchain warnings.\n" + " \"warnings\" : \"...\", (string) any network and blockchain warnings.\n" "}\n" "\nExamples:\n" + HelpExampleCli("getblockchaininfo", "") @@ -1185,6 +1186,7 @@ UniValue getblockchaininfo(const JSONRPCRequest& request) obj.push_back(Pair("difficulty", (double)GetDifficulty())); obj.push_back(Pair("mediantime", (int64_t)chainActive.Tip()->GetMedianTimePast())); obj.push_back(Pair("verificationprogress", GuessVerificationProgress(Params().TxData(), chainActive.Tip()))); + obj.push_back(Pair("initialblockdownload", IsInitialBlockDownload())); obj.push_back(Pair("chainwork", chainActive.Tip()->nChainWork.GetHex())); obj.push_back(Pair("size_on_disk", CalculateCurrentUsage())); obj.push_back(Pair("pruned", fPruneMode)); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index d860dbc244..8f8cdf80ed 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -11,6 +11,7 @@ #include "init.h" #include "keystore.h" #include "validation.h" +#include "validationinterface.h" #include "merkleblock.h" #include "net.h" #include "policy/policy.h" @@ -30,6 +31,7 @@ #include "wallet/wallet.h" #endif +#include <future> #include <stdint.h> #include <univalue.h> @@ -917,7 +919,9 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) ); ObserveSafeMode(); - LOCK(cs_main); + + std::promise<void> promise; + RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL}); // parse hex string from parameter @@ -931,6 +935,8 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) if (!request.params[1].isNull() && request.params[1].get_bool()) nMaxRawTxFee = 0; + { // cs_main scope + LOCK(cs_main); CCoinsViewCache &view = *pcoinsTip; bool fHaveChain = false; for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { @@ -952,10 +958,24 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) } throw JSONRPCError(RPC_TRANSACTION_ERROR, state.GetRejectReason()); } + } else { + // If wallet is enabled, ensure that the wallet has been made aware + // of the new transaction prior to returning. This prevents a race + // where a user might call sendrawtransaction with a transaction + // to/from their wallet, immediately call some wallet RPC, and get + // a stale result because callbacks have not yet been processed. + CallFunctionInValidationInterfaceQueue([&promise] { + promise.set_value(); + }); } } else if (fHaveChain) { throw JSONRPCError(RPC_TRANSACTION_ALREADY_IN_CHAIN, "transaction already in block chain"); } + + } // cs_main + + promise.get_future().wait(); + if(!g_connman) throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); @@ -964,6 +984,7 @@ UniValue sendrawtransaction(const JSONRPCRequest& request) { pnode->PushInventory(inv); }); + return hashTx.GetHex(); } diff --git a/src/sync.cpp b/src/sync.cpp index 87024ccdf2..4ef419cd09 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -155,6 +155,16 @@ void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, abort(); } +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) +{ + for (const std::pair<void*, CLockLocation>& i : *lockstack) { + if (i.first == cs) { + fprintf(stderr, "Assertion failed: lock %s held in %s:%i; locks held:\n%s", pszName, pszFile, nLine, LocksHeld().c_str()); + abort(); + } + } +} + void DeleteLock(void* cs) { if (!lockdata.available) { diff --git a/src/sync.h b/src/sync.h index 20556af890..12160f8dc1 100644 --- a/src/sync.h +++ b/src/sync.h @@ -77,14 +77,17 @@ void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs void LeaveCritical(); std::string LocksHeld(); void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); +void AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); void DeleteLock(void* cs); #else void static inline EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) {} void static inline LeaveCritical() {} void static inline AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} +void static inline AssertLockNotHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} void static inline DeleteLock(void* cs) {} #endif #define AssertLockHeld(cs) AssertLockHeldInternal(#cs, __FILE__, __LINE__, &cs) +#define AssertLockNotHeld(cs) AssertLockNotHeldInternal(#cs, __FILE__, __LINE__, &cs) /** * Wrapped mutex: supports recursive locking, but no waiting diff --git a/src/txmempool.cpp b/src/txmempool.cpp index b0306811cb..c509e22a2e 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -317,7 +317,7 @@ void CTxMemPoolEntry::UpdateDescendantState(int64_t modifySize, CAmount modifyFe assert(int64_t(nCountWithDescendants) > 0); } -void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps) +void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps) { nSizeWithAncestors += modifySize; assert(int64_t(nSizeWithAncestors) > 0); diff --git a/src/txmempool.h b/src/txmempool.h index 929d223588..87f2297c04 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -109,7 +109,7 @@ public: // Adjusts the descendant state. void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state - void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int modifySigOps); + void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); // Updates the fee delta used for mining priority score, and the // modified fees with descendants. void UpdateFeeDelta(int64_t feeDelta); @@ -513,6 +513,9 @@ public: // to track size/count of descendant transactions. First version of // addUnchecked can be used to have it call CalculateMemPoolAncestors(), and // then invoke the second version. + // Note that addUnchecked is ONLY called from ATMP outside of tests + // and any other callers may break wallet's in-mempool tracking (due to + // lack of CValidationInterface::TransactionAddedToMempool callbacks). bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool validFeeEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool validFeeEstimate = true); diff --git a/src/uint256.h b/src/uint256.h index 94a4f7fc30..259161ce0a 100644 --- a/src/uint256.h +++ b/src/uint256.h @@ -19,7 +19,7 @@ template<unsigned int BITS> class base_blob { protected: - enum { WIDTH=BITS/8 }; + static constexpr int WIDTH = BITS / 8; uint8_t data[WIDTH]; public: base_blob() diff --git a/src/validation.cpp b/src/validation.cpp index e7b6fc52a8..c5d73187b7 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2494,7 +2494,7 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, for (const PerBlockConnectTrace& trace : connectTrace.GetBlocksConnected()) { assert(trace.pblock && trace.pindex); - GetMainSignals().BlockConnected(trace.pblock, trace.pindex, *trace.conflictedTxs); + GetMainSignals().BlockConnected(trace.pblock, trace.pindex, trace.conflictedTxs); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 281bc04b0a..d91707cf9f 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -9,6 +9,7 @@ #include "primitives/block.h" #include "scheduler.h" #include "sync.h" +#include "txmempool.h" #include "util.h" #include <list> @@ -21,6 +22,7 @@ struct MainSignalsInstance { boost::signals2::signal<void (const CTransactionRef &)> TransactionAddedToMempool; boost::signals2::signal<void (const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef>&)> BlockConnected; boost::signals2::signal<void (const std::shared_ptr<const CBlock> &)> BlockDisconnected; + boost::signals2::signal<void (const CTransactionRef &)> TransactionRemovedFromMempool; boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; boost::signals2::signal<void (const uint256 &)> Inventory; boost::signals2::signal<void (int64_t nBestBlockTime, CConnman* connman)> Broadcast; @@ -50,6 +52,14 @@ void CMainSignals::FlushBackgroundCallbacks() { m_internals->m_schedulerClient.EmptyQueue(); } +void CMainSignals::RegisterWithMempoolSignals(CTxMemPool& pool) { + pool.NotifyEntryRemoved.connect(boost::bind(&CMainSignals::MempoolEntryRemoved, this, _1, _2)); +} + +void CMainSignals::UnregisterWithMempoolSignals(CTxMemPool& pool) { + pool.NotifyEntryRemoved.disconnect(boost::bind(&CMainSignals::MempoolEntryRemoved, this, _1, _2)); +} + CMainSignals& GetMainSignals() { return g_signals; @@ -60,6 +70,7 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.m_internals->TransactionAddedToMempool.connect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); g_signals.m_internals->BlockConnected.connect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); g_signals.m_internals->BlockDisconnected.connect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); + g_signals.m_internals->TransactionRemovedFromMempool.connect(boost::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, _1)); g_signals.m_internals->SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); g_signals.m_internals->Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); g_signals.m_internals->Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn, _1, _2)); @@ -75,6 +86,7 @@ void UnregisterValidationInterface(CValidationInterface* pwalletIn) { g_signals.m_internals->TransactionAddedToMempool.disconnect(boost::bind(&CValidationInterface::TransactionAddedToMempool, pwalletIn, _1)); g_signals.m_internals->BlockConnected.disconnect(boost::bind(&CValidationInterface::BlockConnected, pwalletIn, _1, _2, _3)); g_signals.m_internals->BlockDisconnected.disconnect(boost::bind(&CValidationInterface::BlockDisconnected, pwalletIn, _1)); + g_signals.m_internals->TransactionRemovedFromMempool.disconnect(boost::bind(&CValidationInterface::TransactionRemovedFromMempool, pwalletIn, _1)); g_signals.m_internals->UpdatedBlockTip.disconnect(boost::bind(&CValidationInterface::UpdatedBlockTip, pwalletIn, _1, _2, _3)); g_signals.m_internals->NewPoWValidBlock.disconnect(boost::bind(&CValidationInterface::NewPoWValidBlock, pwalletIn, _1, _2)); } @@ -87,32 +99,57 @@ void UnregisterAllValidationInterfaces() { g_signals.m_internals->TransactionAddedToMempool.disconnect_all_slots(); g_signals.m_internals->BlockConnected.disconnect_all_slots(); g_signals.m_internals->BlockDisconnected.disconnect_all_slots(); + g_signals.m_internals->TransactionRemovedFromMempool.disconnect_all_slots(); g_signals.m_internals->UpdatedBlockTip.disconnect_all_slots(); g_signals.m_internals->NewPoWValidBlock.disconnect_all_slots(); } +void CallFunctionInValidationInterfaceQueue(std::function<void ()> func) { + g_signals.m_internals->m_schedulerClient.AddToProcessQueue(std::move(func)); +} + +void CMainSignals::MempoolEntryRemoved(CTransactionRef ptx, MemPoolRemovalReason reason) { + if (reason != MemPoolRemovalReason::BLOCK && reason != MemPoolRemovalReason::CONFLICT) { + m_internals->m_schedulerClient.AddToProcessQueue([ptx, this] { + m_internals->TransactionRemovedFromMempool(ptx); + }); + } +} + void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) { - m_internals->UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); + m_internals->m_schedulerClient.AddToProcessQueue([pindexNew, pindexFork, fInitialDownload, this] { + m_internals->UpdatedBlockTip(pindexNew, pindexFork, fInitialDownload); + }); } void CMainSignals::TransactionAddedToMempool(const CTransactionRef &ptx) { - m_internals->TransactionAddedToMempool(ptx); + m_internals->m_schedulerClient.AddToProcessQueue([ptx, this] { + m_internals->TransactionAddedToMempool(ptx); + }); } -void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { - m_internals->BlockConnected(pblock, pindex, vtxConflicted); +void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>>& pvtxConflicted) { + m_internals->m_schedulerClient.AddToProcessQueue([pblock, pindex, pvtxConflicted, this] { + m_internals->BlockConnected(pblock, pindex, *pvtxConflicted); + }); } void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock> &pblock) { - m_internals->BlockDisconnected(pblock); + m_internals->m_schedulerClient.AddToProcessQueue([pblock, this] { + m_internals->BlockDisconnected(pblock); + }); } void CMainSignals::SetBestChain(const CBlockLocator &locator) { - m_internals->SetBestChain(locator); + m_internals->m_schedulerClient.AddToProcessQueue([locator, this] { + m_internals->SetBestChain(locator); + }); } void CMainSignals::Inventory(const uint256 &hash) { - m_internals->Inventory(hash); + m_internals->m_schedulerClient.AddToProcessQueue([hash, this] { + m_internals->Inventory(hash); + }); } void CMainSignals::Broadcast(int64_t nBestBlockTime, CConnman* connman) { diff --git a/src/validationinterface.h b/src/validationinterface.h index d6da2bc1fd..1494c6dc21 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -6,10 +6,11 @@ #ifndef BITCOIN_VALIDATIONINTERFACE_H #define BITCOIN_VALIDATIONINTERFACE_H -#include <memory> - #include "primitives/transaction.h" // CTransaction(Ref) +#include <functional> +#include <memory> + class CBlock; class CBlockIndex; struct CBlockLocator; @@ -20,6 +21,8 @@ class CValidationInterface; class CValidationState; class uint256; class CScheduler; +class CTxMemPool; +enum class MemPoolRemovalReason; // These functions dispatch to one or all registered wallets @@ -29,23 +32,66 @@ void RegisterValidationInterface(CValidationInterface* pwalletIn); void UnregisterValidationInterface(CValidationInterface* pwalletIn); /** Unregister all wallets from core */ void UnregisterAllValidationInterfaces(); +/** + * Pushes a function to callback onto the notification queue, guaranteeing any + * callbacks generated prior to now are finished when the function is called. + * + * Be very careful blocking on func to be called if any locks are held - + * validation interface clients may not be able to make progress as they often + * wait for things like cs_main, so blocking until func is called with cs_main + * will result in a deadlock (that DEBUG_LOCKORDER will miss). + */ +void CallFunctionInValidationInterfaceQueue(std::function<void ()> func); class CValidationInterface { protected: - /** Notifies listeners of updated block chain tip */ + /** + * Notifies listeners of updated block chain tip + * + * Called on a background thread. + */ virtual void UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockIndex *pindexFork, bool fInitialDownload) {} - /** Notifies listeners of a transaction having been added to mempool. */ + /** + * Notifies listeners of a transaction having been added to mempool. + * + * Called on a background thread. + */ virtual void TransactionAddedToMempool(const CTransactionRef &ptxn) {} /** + * Notifies listeners of a transaction leaving mempool. + * + * This only fires for transactions which leave mempool because of expiry, + * size limiting, reorg (changes in lock times/coinbase maturity), or + * replacement. This does not include any transactions which are included + * in BlockConnectedDisconnected either in block->vtx or in txnConflicted. + * + * Called on a background thread. + */ + virtual void TransactionRemovedFromMempool(const CTransactionRef &ptx) {} + /** * Notifies listeners of a block being connected. * Provides a vector of transactions evicted from the mempool as a result. + * + * Called on a background thread. */ virtual void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex, const std::vector<CTransactionRef> &txnConflicted) {} - /** Notifies listeners of a block being disconnected */ + /** + * Notifies listeners of a block being disconnected + * + * Called on a background thread. + */ virtual void BlockDisconnected(const std::shared_ptr<const CBlock> &block) {} - /** Notifies listeners of the new active block chain on-disk. */ + /** + * Notifies listeners of the new active block chain on-disk. + * + * Called on a background thread. + */ virtual void SetBestChain(const CBlockLocator &locator) {} - /** Notifies listeners about an inventory item being seen on the network. */ + /** + * Notifies listeners about an inventory item being seen on the network. + * + * Called on a background thread. + */ virtual void Inventory(const uint256 &hash) {} /** Tells listeners to broadcast their data. */ virtual void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) {} @@ -73,6 +119,9 @@ private: friend void ::RegisterValidationInterface(CValidationInterface*); friend void ::UnregisterValidationInterface(CValidationInterface*); friend void ::UnregisterAllValidationInterfaces(); + friend void ::CallFunctionInValidationInterfaceQueue(std::function<void ()> func); + + void MempoolEntryRemoved(CTransactionRef tx, MemPoolRemovalReason reason); public: /** Register a CScheduler to give callbacks which should run in the background (may only be called once) */ @@ -82,9 +131,14 @@ public: /** Call any remaining callbacks on the calling thread */ void FlushBackgroundCallbacks(); + /** Register with mempool to call TransactionRemovedFromMempool callbacks */ + void RegisterWithMempoolSignals(CTxMemPool& pool); + /** Unregister with mempool */ + void UnregisterWithMempoolSignals(CTxMemPool& pool); + void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); void TransactionAddedToMempool(const CTransactionRef &); - void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::vector<CTransactionRef> &); + void BlockConnected(const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex, const std::shared_ptr<const std::vector<CTransactionRef>> &); void BlockDisconnected(const std::shared_ptr<const CBlock> &); void SetBestChain(const CBlockLocator &); void Inventory(const uint256 &); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 2d2eb4b6d3..b081803be6 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -455,6 +455,11 @@ UniValue sendtoaddress(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); CTxDestination dest = DecodeDestination(request.params[0].get_str()); @@ -533,6 +538,11 @@ UniValue listaddressgroupings(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); UniValue jsonGroupings(UniValue::VARR); @@ -645,6 +655,11 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); // Bitcoin address @@ -654,7 +669,7 @@ UniValue getreceivedbyaddress(const JSONRPCRequest& request) } CScript scriptPubKey = GetScriptForDestination(dest); if (!IsMine(*pwallet, scriptPubKey)) { - return ValueFromAmount(0); + throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); } // Minimum confirmations @@ -707,6 +722,11 @@ UniValue getreceivedbyaccount(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); // Minimum confirmations @@ -780,6 +800,11 @@ UniValue getbalance(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); const UniValue& account_value = request.params[0]; @@ -825,6 +850,11 @@ UniValue getunconfirmedbalance(const JSONRPCRequest &request) "Returns the server's total unconfirmed balance\n"); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); return ValueFromAmount(pwallet->GetUnconfirmedBalance()); @@ -919,6 +949,11 @@ UniValue sendfrom(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); std::string strAccount = AccountFromValue(request.params[0]); @@ -1004,6 +1039,11 @@ UniValue sendmany(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); if (pwallet->GetBroadcastTransactions() && !g_connman) { @@ -1455,6 +1495,11 @@ UniValue listreceivedbyaddress(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); return ListReceived(pwallet, request.params, false); @@ -1495,6 +1540,11 @@ UniValue listreceivedbyaccount(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); return ListReceived(pwallet, request.params, true); @@ -1683,6 +1733,11 @@ UniValue listtransactions(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); std::string strAccount = "*"; @@ -1777,6 +1832,11 @@ UniValue listaccounts(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); int nMinDepth = 1; @@ -1886,6 +1946,11 @@ UniValue listsinceblock(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); const CBlockIndex* pindex = nullptr; // Block index of the specified block or the common ancestor, if the block provided was in a deactivated chain. @@ -2019,6 +2084,11 @@ UniValue gettransaction(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); uint256 hash; @@ -2081,6 +2151,11 @@ UniValue abandontransaction(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); uint256 hash; @@ -2115,6 +2190,10 @@ UniValue backupwallet(const JSONRPCRequest& request) + HelpExampleRpc("backupwallet", "\"backup.dat\"") ); + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); std::string strDest = request.params[0].get_str(); @@ -2434,6 +2513,10 @@ UniValue lockunspent(const JSONRPCRequest& request) + HelpExampleRpc("lockunspent", "false, \"[{\\\"txid\\\":\\\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\\\",\\\"vout\\\":1}]\"") ); + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); RPCTypeCheckArgument(request.params[0], UniValue::VBOOL); @@ -2593,6 +2676,11 @@ UniValue getwalletinfo(const JSONRPCRequest& request) ); ObserveSafeMode(); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); UniValue obj(UniValue::VOBJ); @@ -2802,9 +2890,12 @@ UniValue listunspent(const JSONRPCRequest& request) nMaximumCount = options["maximumCount"].get_int64(); } + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + UniValue results(UniValue::VARR); std::vector<COutput> vecOutputs; - assert(pwallet != nullptr); LOCK2(cs_main, pwallet->cs_wallet); pwallet->AvailableCoins(vecOutputs, !include_unsafe, nullptr, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount, nMinDepth, nMaxDepth); @@ -2912,6 +3003,10 @@ UniValue fundrawtransaction(const JSONRPCRequest& request) ObserveSafeMode(); RPCTypeCheck(request.params, {UniValue::VSTR}); + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + CCoinControl coinControl; int changePosition = -1; bool lockUnspents = false; @@ -3122,6 +3217,10 @@ UniValue bumpfee(const JSONRPCRequest& request) } } + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + LOCK2(cs_main, pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index dceb818b50..8eb84d397f 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -33,6 +33,7 @@ #include "wallet/fees.h" #include <assert.h> +#include <future> #include <boost/algorithm/string/replace.hpp> #include <boost/thread.hpp> @@ -1217,6 +1218,19 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, const CBlockIndex *pin void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { LOCK2(cs_main, cs_wallet); SyncTransaction(ptx); + + auto it = mapWallet.find(ptx->GetHash()); + if (it != mapWallet.end()) { + it->second.fInMempool = true; + } +} + +void CWallet::TransactionRemovedFromMempool(const CTransactionRef &ptx) { + LOCK(cs_wallet); + auto it = mapWallet.find(ptx->GetHash()); + if (it != mapWallet.end()) { + it->second.fInMempool = false; + } } void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { @@ -1231,10 +1245,14 @@ void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const for (const CTransactionRef& ptx : vtxConflicted) { SyncTransaction(ptx); + TransactionRemovedFromMempool(ptx); } for (size_t i = 0; i < pblock->vtx.size(); i++) { SyncTransaction(pblock->vtx[i], pindex, i); + TransactionRemovedFromMempool(pblock->vtx[i]); } + + m_last_block_processed = pindex; } void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { @@ -1247,6 +1265,36 @@ void CWallet::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) { +void CWallet::BlockUntilSyncedToCurrentChain() { + AssertLockNotHeld(cs_main); + AssertLockNotHeld(cs_wallet); + + { + // Skip the queue-draining stuff if we know we're caught up with + // chainActive.Tip()... + // We could also take cs_wallet here, and call m_last_block_processed + // protected by cs_wallet instead of cs_main, but as long as we need + // cs_main here anyway, its easier to just call it cs_main-protected. + LOCK(cs_main); + const CBlockIndex* initialChainTip = chainActive.Tip(); + + if (m_last_block_processed->GetAncestor(initialChainTip->nHeight) == initialChainTip) { + return; + } + } + + // ...otherwise put a callback in the validation interface queue and wait + // for the queue to drain enough to execute it (indicating we are caught up + // at least with the time we entered this function). + + std::promise<void> promise; + CallFunctionInValidationInterfaceQueue([&promise] { + promise.set_value(); + }); + promise.get_future().wait(); +} + + isminetype CWallet::IsMine(const CTxIn &txin) const { { @@ -1871,8 +1919,7 @@ CAmount CWalletTx::GetChange() const bool CWalletTx::InMempool() const { - LOCK(mempool.cs); - return mempool.exists(GetHash()); + return fInMempool; } bool CWalletTx::IsTrusted() const @@ -2980,14 +3027,18 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon // Track how many getdata requests our transaction gets mapRequestCount[wtxNew.GetHash()] = 0; + // Get the inserted-CWalletTx from mapWallet so that the + // fInMempool flag is cached properly + CWalletTx& wtx = mapWallet[wtxNew.GetHash()]; + if (fBroadcastTransactions) { // Broadcast - if (!wtxNew.AcceptToMemoryPool(maxTxFee, state)) { + if (!wtx.AcceptToMemoryPool(maxTxFee, state)) { LogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", state.GetRejectReason()); // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. } else { - wtxNew.RelayWalletTransaction(connman); + wtx.RelayWalletTransaction(connman); } } } @@ -3903,8 +3954,6 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) LogPrintf(" wallet %15dms\n", GetTimeMillis() - nStart); - RegisterValidationInterface(walletInstance); - // Try to top up keypool. No-op if the wallet is locked. walletInstance->TopUpKeyPool(); @@ -3916,6 +3965,10 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) if (walletdb.ReadBestBlock(locator)) pindexRescan = FindForkInGlobalIndex(chainActive, locator); } + + walletInstance->m_last_block_processed = chainActive.Tip(); + RegisterValidationInterface(walletInstance); + if (chainActive.Tip() && chainActive.Tip() != pindexRescan) { //We can't rescan beyond non-pruned blocks, stop and throw an error @@ -4059,8 +4112,15 @@ int CMerkleTx::GetBlocksToMaturity() const } -bool CMerkleTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) +bool CWalletTx::AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state) { - return ::AcceptToMemoryPool(mempool, state, tx, nullptr /* pfMissingInputs */, + // We must set fInMempool here - while it will be re-set to true by the + // entered-mempool callback, if we did not there would be a race where a + // user could call sendmoney in a loop and hit spurious out of funds errors + // because we think that the transaction they just generated's change is + // unavailable as we're not yet aware its in mempool. + bool ret = ::AcceptToMemoryPool(mempool, state, tx, nullptr /* pfMissingInputs */, nullptr /* plTxnReplaced */, false /* bypass_limits */, nAbsurdFee); + fInMempool = ret; + return ret; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index db2861c043..c372f3ba19 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -248,8 +248,6 @@ public: int GetDepthInMainChain() const { const CBlockIndex *pindexRet; return GetDepthInMainChain(pindexRet); } bool IsInMainChain() const { const CBlockIndex *pindexRet; return GetDepthInMainChain(pindexRet) > 0; } int GetBlocksToMaturity() const; - /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ - bool AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state); bool hashUnset() const { return (hashBlock.IsNull() || hashBlock == ABANDON_HASH); } bool isAbandoned() const { return (hashBlock == ABANDON_HASH); } void setAbandoned() { hashBlock = ABANDON_HASH; } @@ -326,6 +324,7 @@ public: mutable bool fImmatureWatchCreditCached; mutable bool fAvailableWatchCreditCached; mutable bool fChangeCached; + mutable bool fInMempool; mutable CAmount nDebitCached; mutable CAmount nCreditCached; mutable CAmount nImmatureCreditCached; @@ -365,6 +364,7 @@ public: fImmatureWatchCreditCached = false; fAvailableWatchCreditCached = false; fChangeCached = false; + fInMempool = false; nDebitCached = 0; nCreditCached = 0; nImmatureCreditCached = 0; @@ -469,6 +469,9 @@ public: // RelayWalletTransaction may only be called if fBroadcastTransactions! bool RelayWalletTransaction(CConnman* connman); + /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ + bool AcceptToMemoryPool(const CAmount& nAbsurdFee, CValidationState& state); + std::set<uint256> GetConflicts() const; }; @@ -718,6 +721,18 @@ private: std::unique_ptr<CWalletDBWrapper> dbw; + /** + * The following is used to keep track of how far behind the wallet is + * from the chain sync, and to allow clients to block on us being caught up. + * + * Note that this is *not* how far we've processed, we may need some rescan + * to have seen all transactions in the chain, but is only used to track + * live BlockConnected callbacks. + * + * Protected by cs_main (see BlockUntilSyncedToCurrentChain) + */ + const CBlockIndex* m_last_block_processed; + public: /* * Main wallet lock. @@ -916,6 +931,7 @@ public: bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate); int64_t RescanFromTime(int64_t startTime, bool update); CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop, bool fUpdate = false); + void TransactionRemovedFromMempool(const CTransactionRef &ptx) override; void ReacceptWalletTransactions(); void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override; // ResendWalletTransactionsBefore may only be called if fBroadcastTransactions! @@ -1102,6 +1118,14 @@ public: caller must ensure the current wallet version is correct before calling this function). */ bool SetHDMasterKey(const CPubKey& key); + + /** + * Blocks until the wallet state is up-to-date to /at least/ the current + * chain at the time this function is entered + * Obviously holding cs_main/cs_wallet when going into this call may cause + * deadlock + */ + void BlockUntilSyncedToCurrentChain(); }; /** A key allocated from the key pool. */ diff --git a/test/functional/blockchain.py b/test/functional/blockchain.py index 4576cb036a..49fafbc9aa 100755 --- a/test/functional/blockchain.py +++ b/test/functional/blockchain.py @@ -5,6 +5,7 @@ """Test RPCs related to blockchainstate. Test the following RPCs: + - getblockchaininfo - gettxoutsetinfo - getdifficulty - getbestblockhash @@ -58,6 +59,7 @@ class BlockchainTest(BitcoinTestFramework): 'chainwork', 'difficulty', 'headers', + 'initialblockdownload', 'mediantime', 'pruned', 'size_on_disk', diff --git a/test/functional/maxuploadtarget.py b/test/functional/maxuploadtarget.py index 9c92aa1dc0..88e2ff2e16 100755 --- a/test/functional/maxuploadtarget.py +++ b/test/functional/maxuploadtarget.py @@ -54,7 +54,7 @@ class MaxUploadTest(BitcoinTestFramework): # p2p_conns[2] will test resetting the counters p2p_conns = [] - for i in range(3): + for _ in range(3): p2p_conns.append(self.nodes[0].add_p2p_connection(TestNode())) NetworkThread().start() # Start up network handling in another thread @@ -139,8 +139,7 @@ class MaxUploadTest(BitcoinTestFramework): self.log.info("Peer 2 able to download old block") - for i in range(3): - self.nodes[0].disconnect_p2p() + self.nodes[0].disconnect_p2ps() #stop and start node 0 with 1MB maxuploadtarget, whitelist 127.0.0.1 self.log.info("Restarting nodes with -whitelist=127.0.0.1") diff --git a/test/functional/p2p-acceptblock.py b/test/functional/p2p-acceptblock.py index fbe5a78029..ca0e0080a1 100755 --- a/test/functional/p2p-acceptblock.py +++ b/test/functional/p2p-acceptblock.py @@ -206,7 +206,7 @@ class AcceptBlockTest(BitcoinTestFramework): # The node should have requested the blocks at some point, so # disconnect/reconnect first - self.nodes[0].disconnect_p2p() + self.nodes[0].disconnect_p2ps() test_node = self.nodes[0].add_p2p_connection(NodeConnCB()) test_node.wait_for_verack() @@ -291,7 +291,7 @@ class AcceptBlockTest(BitcoinTestFramework): except AssertionError: test_node.wait_for_disconnect() - self.nodes[0].disconnect_p2p() + self.nodes[0].disconnect_p2ps() test_node = self.nodes[0].add_p2p_connection(NodeConnCB()) NetworkThread().start() # Start up network handling in another thread diff --git a/test/functional/p2p-leaktests.py b/test/functional/p2p-leaktests.py index a6e47b5df6..719a03914d 100755 --- a/test/functional/p2p-leaktests.py +++ b/test/functional/p2p-leaktests.py @@ -125,8 +125,7 @@ class P2PLeakTest(BitcoinTestFramework): assert not unsupported_service_bit5_node.connected assert not unsupported_service_bit7_node.connected - for _ in range(5): - self.nodes[0].disconnect_p2p() + self.nodes[0].disconnect_p2ps() # Wait until all connections are closed wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0) diff --git a/test/functional/p2p-segwit.py b/test/functional/p2p-segwit.py index 22da7f2db1..b940bc4096 100755 --- a/test/functional/p2p-segwit.py +++ b/test/functional/p2p-segwit.py @@ -32,9 +32,10 @@ def get_virtual_size(witness_block): return vsize class TestNode(NodeConnCB): - def __init__(self): + def __init__(self, rpc): super().__init__() self.getdataset = set() + self.rpc = rpc def on_getdata(self, conn, message): for inv in message.inv: @@ -73,7 +74,7 @@ class TestNode(NodeConnCB): tx_message = msg_witness_tx(tx) self.send_message(tx_message) self.sync_with_ping() - assert_equal(tx.hash in self.connection.rpc.getrawmempool(), accepted) + assert_equal(tx.hash in self.rpc.getrawmempool(), accepted) if (reason != None and not accepted): # Check the rejection reason as well. with mininode_lock: @@ -86,7 +87,7 @@ class TestNode(NodeConnCB): else: self.send_message(msg_block(block)) self.sync_with_ping() - assert_equal(self.connection.rpc.getbestblockhash() == block.hash, accepted) + assert_equal(self.rpc.getbestblockhash() == block.hash, accepted) # Used to keep track of anyone-can-spend outputs that we can use in the tests class UTXO(): @@ -1869,11 +1870,11 @@ class SegWitTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections and start up the network thread. # self.test_node sets NODE_WITNESS|NODE_NETWORK - self.test_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + self.test_node = self.nodes[0].add_p2p_connection(TestNode(self.nodes[0].rpc), services=NODE_NETWORK|NODE_WITNESS) # self.old_node sets only NODE_NETWORK - self.old_node = self.nodes[0].add_p2p_connection(TestNode(), services=NODE_NETWORK) + self.old_node = self.nodes[0].add_p2p_connection(TestNode(self.nodes[0].rpc), services=NODE_NETWORK) # self.std_node is for testing node1 (fRequireStandard=true) - self.std_node = self.nodes[1].add_p2p_connection(TestNode(), services=NODE_NETWORK|NODE_WITNESS) + self.std_node = self.nodes[1].add_p2p_connection(TestNode(self.nodes[1].rpc), services=NODE_NETWORK|NODE_WITNESS) NetworkThread().start() # Start up network handling in another thread diff --git a/test/functional/receivedby.py b/test/functional/receivedby.py index db6fc86b82..97da19546f 100755 --- a/test/functional/receivedby.py +++ b/test/functional/receivedby.py @@ -3,97 +3,83 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listreceivedbyaddress RPC.""" +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -def get_sub_array_from_array(object_array, to_match): - ''' - Finds and returns a sub array from an array of arrays. - to_match should be a unique idetifier of a sub array - ''' - for item in object_array: - all_match = True - for key,value in to_match.items(): - if item[key] != value: - all_match = False - if not all_match: - continue - return item - return [] +from test_framework.util import (assert_array_result, + assert_equal, + assert_raises_rpc_error, + ) class ReceivedByTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.enable_mocktime() def run_test(self): - ''' - listreceivedbyaddress Test - ''' + # Generate block to get out of IBD + self.nodes[0].generate(1) + + self.log.info("listreceivedbyaddress Test") + # Send from node 0 to 1 addr = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(addr, 0.1) self.sync_all() - #Check not listed in listreceivedbyaddress because has 0 confirmations + # Check not listed in listreceivedbyaddress because has 0 confirmations assert_array_result(self.nodes[1].listreceivedbyaddress(), - {"address":addr}, - { }, - True) - #Bury Tx under 10 block so it will be returned by listreceivedbyaddress + {"address": addr}, + {}, + True) + # Bury Tx under 10 block so it will be returned by listreceivedbyaddress self.nodes[1].generate(10) self.sync_all() assert_array_result(self.nodes[1].listreceivedbyaddress(), - {"address":addr}, - {"address":addr, "account":"", "amount":Decimal("0.1"), "confirmations":10, "txids":[txid,]}) - #With min confidence < 10 + {"address": addr}, + {"address": addr, "account": "", "amount": Decimal("0.1"), "confirmations": 10, "txids": [txid, ]}) + # With min confidence < 10 assert_array_result(self.nodes[1].listreceivedbyaddress(5), - {"address":addr}, - {"address":addr, "account":"", "amount":Decimal("0.1"), "confirmations":10, "txids":[txid,]}) - #With min confidence > 10, should not find Tx - assert_array_result(self.nodes[1].listreceivedbyaddress(11),{"address":addr},{ },True) + {"address": addr}, + {"address": addr, "account": "", "amount": Decimal("0.1"), "confirmations": 10, "txids": [txid, ]}) + # With min confidence > 10, should not find Tx + assert_array_result(self.nodes[1].listreceivedbyaddress(11), {"address": addr}, {}, True) - #Empty Tx + # Empty Tx addr = self.nodes[1].getnewaddress() - assert_array_result(self.nodes[1].listreceivedbyaddress(0,True), - {"address":addr}, - {"address":addr, "account":"", "amount":0, "confirmations":0, "txids":[]}) + assert_array_result(self.nodes[1].listreceivedbyaddress(0, True), + {"address": addr}, + {"address": addr, "account": "", "amount": 0, "confirmations": 0, "txids": []}) + + self.log.info("getreceivedbyaddress Test") - ''' - getreceivedbyaddress Test - ''' # Send from node 0 to 1 addr = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(addr, 0.1) self.sync_all() - #Check balance is 0 because of 0 confirmations + # Check balance is 0 because of 0 confirmations balance = self.nodes[1].getreceivedbyaddress(addr) - if balance != Decimal("0.0"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.0")) - #Check balance is 0.1 - balance = self.nodes[1].getreceivedbyaddress(addr,0) - if balance != Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + # Check balance is 0.1 + balance = self.nodes[1].getreceivedbyaddress(addr, 0) + assert_equal(balance, Decimal("0.1")) - #Bury Tx under 10 block so it will be returned by the default getreceivedbyaddress + # Bury Tx under 10 block so it will be returned by the default getreceivedbyaddress self.nodes[1].generate(10) self.sync_all() balance = self.nodes[1].getreceivedbyaddress(addr) - if balance != Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.1")) + + # Trying to getreceivedby for an address the wallet doesn't own should return an error + assert_raises_rpc_error(-4, "Address not found in wallet", self.nodes[0].getreceivedbyaddress, addr) + + self.log.info("listreceivedbyaccount + getreceivedbyaccount Test") - ''' - listreceivedbyaccount + getreceivedbyaccount Test - ''' - #set pre-state + # set pre-state addrArr = self.nodes[1].getnewaddress() account = self.nodes[1].getaccount(addrArr) - received_by_account_json = get_sub_array_from_array(self.nodes[1].listreceivedbyaccount(),{"account":account}) - if len(received_by_account_json) == 0: - raise AssertionError("No accounts found in node") + received_by_account_json = [r for r in self.nodes[1].listreceivedbyaccount() if r["account"] == account][0] balance_by_account = self.nodes[1].getreceivedbyaccount(account) txid = self.nodes[0].sendtoaddress(addr, 0.1) @@ -101,40 +87,34 @@ class ReceivedByTest(BitcoinTestFramework): # listreceivedbyaccount should return received_by_account_json because of 0 confirmations assert_array_result(self.nodes[1].listreceivedbyaccount(), - {"account":account}, - received_by_account_json) + {"account": account}, + received_by_account_json) # getreceivedbyaddress should return same balance because of 0 confirmations balance = self.nodes[1].getreceivedbyaccount(account) - if balance != balance_by_account: - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, balance_by_account) self.nodes[1].generate(10) self.sync_all() # listreceivedbyaccount should return updated account balance assert_array_result(self.nodes[1].listreceivedbyaccount(), - {"account":account}, - {"account":received_by_account_json["account"], "amount":(received_by_account_json["amount"] + Decimal("0.1"))}) + {"account": account}, + {"account": received_by_account_json["account"], "amount": (received_by_account_json["amount"] + Decimal("0.1"))}) # getreceivedbyaddress should return updates balance balance = self.nodes[1].getreceivedbyaccount(account) - if balance != balance_by_account + Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, balance_by_account + Decimal("0.1")) - #Create a new account named "mynewaccount" that has a 0 balance + # Create a new account named "mynewaccount" that has a 0 balance self.nodes[1].getaccountaddress("mynewaccount") - received_by_account_json = get_sub_array_from_array(self.nodes[1].listreceivedbyaccount(0,True),{"account":"mynewaccount"}) - if len(received_by_account_json) == 0: - raise AssertionError("No accounts found in node") + received_by_account_json = [r for r in self.nodes[1].listreceivedbyaccount(0, True) if r["account"] == "mynewaccount"][0] # Test includeempty of listreceivedbyaccount - if received_by_account_json["amount"] != Decimal("0.0"): - raise AssertionError("Wrong balance returned by listreceivedbyaccount, %0.2f"%(received_by_account_json["amount"])) + assert_equal(received_by_account_json["amount"], Decimal("0.0")) # Test getreceivedbyaccount for 0 amount accounts balance = self.nodes[1].getreceivedbyaccount("mynewaccount") - if balance != Decimal("0.0"): - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.0")) if __name__ == '__main__': ReceivedByTest().main() diff --git a/test/functional/test_framework/comptool.py b/test/functional/test_framework/comptool.py index 03f967ba71..723826bae4 100755 --- a/test/functional/test_framework/comptool.py +++ b/test/functional/test_framework/comptool.py @@ -177,7 +177,7 @@ class TestManager(): # Create a p2p connection to each node test_node = TestNode(self.block_store, self.tx_store) self.test_nodes.append(test_node) - self.connections.append(NodeConn('127.0.0.1', p2p_port(i), nodes[i], test_node)) + self.connections.append(NodeConn('127.0.0.1', p2p_port(i), test_node)) # Make sure the TestNode (callback class) has a reference to its # associated NodeConn test_node.add_connection(self.connections[-1]) diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 8fbc63fba4..24ee09b81c 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1420,7 +1420,6 @@ class NodeConnCB(): conn.send_message(msg_pong(message.nonce)) def on_verack(self, conn, message): - conn.ver_recv = conn.ver_send self.verack_received = True def on_version(self, conn, message): @@ -1516,7 +1515,7 @@ class NodeConn(asyncore.dispatcher): "regtest": b"\xfa\xbf\xb5\xda", # regtest } - def __init__(self, dstaddr, dstport, rpc, callback, net="regtest", services=NODE_NETWORK|NODE_WITNESS, send_version=True): + def __init__(self, dstaddr, dstport, callback, net="regtest", services=NODE_NETWORK|NODE_WITNESS, send_version=True): asyncore.dispatcher.__init__(self, map=mininode_socket_map) self.dstaddr = dstaddr self.dstport = dstport @@ -1524,8 +1523,6 @@ class NodeConn(asyncore.dispatcher): self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.sendbuf = b"" self.recvbuf = b"" - self.ver_send = 209 - self.ver_recv = 209 self.last_sent = 0 self.state = "connecting" self.network = net @@ -1549,7 +1546,6 @@ class NodeConn(asyncore.dispatcher): self.connect((dstaddr, dstport)) except: self.handle_close() - self.rpc = rpc def handle_connect(self): if self.state != "connected": diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 8b28064c46..34b458482a 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -168,7 +168,7 @@ class TestNode(): if 'dstaddr' not in kwargs: kwargs['dstaddr'] = '127.0.0.1' self.p2ps.append(p2p_conn) - kwargs.update({'rpc': self.rpc, 'callback': p2p_conn}) + kwargs.update({'callback': p2p_conn}) p2p_conn.add_connection(NodeConn(**kwargs)) return p2p_conn @@ -182,13 +182,14 @@ class TestNode(): assert self.p2ps, "No p2p connection" return self.p2ps[0] - def disconnect_p2p(self, index=0): - """Close the p2p connection to the node.""" - # Connection could have already been closed by other end. Calling disconnect_p2p() - # on an already disconnected p2p connection is not an error. - if self.p2ps[index].connection is not None: - self.p2ps[index].connection.disconnect_node() - del self.p2ps[index] + def disconnect_p2ps(self): + """Close all p2p connections to the node.""" + for p in self.p2ps: + # Connection could have already been closed by other end. + if p.connection is not None: + p.connection.disconnect_node() + self.p2ps = [] + class TestNodeCLI(): """Interface to bitcoin-cli for an individual node""" |