diff options
92 files changed, 775 insertions, 257 deletions
diff --git a/.travis.yml b/.travis.yml index cc5d2d2161..9332160926 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,7 @@ env: # x86_64 Linux, No wallet - HOST=x86_64-unknown-linux-gnu PACKAGES="python3" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" # Cross-Mac - - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" OSX_SDK=10.11 GOAL="deploy" + - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" OSX_SDK=10.11 GOAL="all deploy" before_install: - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c390595abf..3ee5a04796 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -127,10 +127,10 @@ before it will be merged. The basic squashing workflow is shown below. git checkout your_branch_name git rebase -i HEAD~n - # n is normally the number of commits in the pull - # set commits from 'pick' to 'squash', save and quit - # on the next screen, edit/refine commit messages - # save and quit + # n is normally the number of commits in the pull request. + # Set commits (except the one in the first line) from 'pick' to 'squash', save and quit. + # On the next screen, edit/refine commit messages. + # Save and quit. git push -f # (force push to GitHub) If you have problems with squashing (or other workflows with `git`), you can diff --git a/Makefile.am b/Makefile.am index 0ed3dd289a..f3f3302fce 100644 --- a/Makefile.am +++ b/Makefile.am @@ -95,9 +95,9 @@ $(OSX_APP)/Contents/Resources/bitcoin.icns: $(OSX_INSTALLER_ICONS) $(MKDIR_P) $(@D) $(INSTALL_DATA) $< $@ -$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(BITCOIN_QT_BIN) +$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: all-recursive $(MKDIR_P) $(@D) - STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $< $@ + STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $@ $(OSX_APP)/Contents/Resources/Base.lproj/InfoPlist.strings: $(MKDIR_P) $(@D) diff --git a/autogen.sh b/autogen.sh index 27417daf76..0c05626ccc 100755 --- a/autogen.sh +++ b/autogen.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C set -e srcdir="$(dirname $0)" cd "$srcdir" diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index d544d5c371..abfa5ed5ae 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -37,7 +37,7 @@ for arg in sys.argv[1:]: # TODO: implement support for multiple include directories for arg in sorted(files.keys()): module = files[arg] - with open(arg, 'r') as f: + with open(arg, 'r', encoding="utf8") as f: for line in f: match = RE.match(line) if match: diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 5402870fba..77e845a9b4 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -152,7 +152,7 @@ def main(): sys.exit(p.returncode) if not args.i: - with open(filename) as f: + with open(filename, encoding="utf8") as f: code = f.readlines() formatted_code = io.StringIO(stdout).readlines() diff = difflib.unified_diff(code, formatted_code, diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 82d3c19683..da7d74bdc4 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -146,7 +146,7 @@ def file_has_without_c_style_copyright_for_holder(contents, holder_name): ################################################################################ def read_file(filename): - return open(os.path.abspath(filename), 'r').read() + return open(os.path.abspath(filename), 'r', encoding="utf8").read() def gather_file_info(filename): info = {} @@ -325,13 +325,13 @@ def get_most_recent_git_change_year(filename): ################################################################################ def read_file_lines(filename): - f = open(os.path.abspath(filename), 'r') + f = open(os.path.abspath(filename), 'r', encoding="utf8") file_lines = f.readlines() f.close() return file_lines def write_file_lines(filename, file_lines): - f = open(os.path.abspath(filename), 'w') + f = open(os.path.abspath(filename), 'w', encoding="utf8") f.write(''.join(file_lines)) f.close() diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index 27c80548c1..76d4adc81c 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -1,5 +1,6 @@ #!/bin/bash +export LC_ALL=C TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} BUILDDIR=${BUILDDIR:-$TOPDIR} diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 187ef75fb7..4e90f85f50 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -191,7 +191,7 @@ def main(): merge_branch = 'pull/'+pull+'/merge' local_merge_branch = 'pull/'+pull+'/local-merge' - devnull = open(os.devnull,'w') + devnull = open(os.devnull, 'w', encoding="utf8") try: subprocess.check_call([GIT,'checkout','-q',branch]) except subprocess.CalledProcessError: diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 307e057736..9b6d6bf665 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -9,7 +9,7 @@ import subprocess import unittest def write_testcode(filename): - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: f.write(''' #include <stdio.h> int main() diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py index 299377d691..df1db76e92 100755 --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -13,8 +13,8 @@ pattern = args.pattern outfile = args.outfile in_remove = False -with open(tracefile, 'r') as f: - with open(outfile, 'w') as wf: +with open(tracefile, 'r', encoding="utf8") as f: + with open(outfile, 'w', encoding="utf8") as wf: for line in f: for p in pattern: if line.startswith("SF:") and p in line: diff --git a/contrib/gitian-build.sh b/contrib/gitian-build.sh index 5a925f2282..8bfbf6d1a0 100755 --- a/contrib/gitian-build.sh +++ b/contrib/gitian-build.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C # What to do sign=false verify=false diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index d315a7d3b7..4f74e67f2f 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -2,6 +2,7 @@ # Install libdb4.8 (Berkeley DB). +export LC_ALL=C set -e if [ -z "${1}" ]; then diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index f8aea27342..b501388fd2 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -75,7 +75,7 @@ def get_blk_dt(blk_hdr): # When getting the list of block hashes, undo any byte reversals. def get_block_hashes(settings): blkindex = [] - f = open(settings['hashlist'], "r") + f = open(settings['hashlist'], "r", encoding="utf8") for line in f: line = line.rstrip() if settings['rev_hash_bytes'] == 'true': @@ -261,7 +261,7 @@ if __name__ == '__main__': print("Usage: linearize-data.py CONFIG-FILE") sys.exit(1) - f = open(sys.argv[1]) + f = open(sys.argv[1], encoding="utf8") for line in f: # skip comment lines m = re.search('^\s*#', line) diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 8e1266ae0b..bfd2171947 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -96,7 +96,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000): def get_rpc_cookie(): # Open the cookie file - with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f: + with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r', encoding="ascii") as f: combined = f.readline() combined_split = combined.split(":") settings['rpcuser'] = combined_split[0] @@ -107,7 +107,7 @@ if __name__ == '__main__': print("Usage: linearize-hashes.py CONFIG-FILE") sys.exit(1) - f = open(sys.argv[1]) + f = open(sys.argv[1], encoding="utf8") for line in f: # skip comment lines m = re.search('^\s*#', line) diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh index 91674a92e6..f8503e4de8 100755 --- a/contrib/macdeploy/detached-sig-apply.sh +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C set -e UNSIGNED="$1" diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 3379a4599c..5281ebcc47 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C set -e ROOTDIR=dist diff --git a/contrib/macdeploy/extract-osx-sdk.sh b/contrib/macdeploy/extract-osx-sdk.sh index ff9fbd58df..94122b56fc 100755 --- a/contrib/macdeploy/extract-osx-sdk.sh +++ b/contrib/macdeploy/extract-osx-sdk.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C set -e INPUTFILE="Xcode_7.3.1.dmg" diff --git a/contrib/qos/tc.sh b/contrib/qos/tc.sh index 0d1dd65b4f..738ea70dbe 100644 --- a/contrib/qos/tc.sh +++ b/contrib/qos/tc.sh @@ -2,6 +2,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C #network interface on which to limit traffic IF="eth0" #limit of the network interface in question diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index ace7d3534f..fe7cd1d597 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -127,10 +127,10 @@ def main(): g.write(' * Each line contains a 16-byte IPv6 address and a port.\n') g.write(' * IPv4 as well as onion addresses are wrapped inside an IPv6 address accordingly.\n') g.write(' */\n') - with open(os.path.join(indir,'nodes_main.txt'),'r') as f: + with open(os.path.join(indir,'nodes_main.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'pnSeed6_main', 8333) g.write('\n') - with open(os.path.join(indir,'nodes_test.txt'),'r') as f: + with open(os.path.join(indir,'nodes_test.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'pnSeed6_test', 18333) g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n') diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index 8f3e4b8063..7a10ba7d7d 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C INPUT=$(cat /dev/stdin) VALID=false REVSIG=false @@ -57,7 +58,7 @@ if ! $VALID; then exit 1 fi if $VALID && $REVSIG; then - printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "^\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" echo "$GOODREVSIG" else printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh index 1f14635f61..152f4e32e4 100755 --- a/contrib/verify-commits/pre-push-hook.sh +++ b/contrib/verify-commits/pre-push-hook.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C if ! [[ "$2" =~ ^(git@)?(www.)?github.com(:|/)bitcoin/bitcoin(.git)?$ ]]; then exit 0 fi diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 80f0aa0bf1..a9e4977715 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -76,11 +76,11 @@ def main(): # get directory of this program and read data files dirname = os.path.dirname(os.path.abspath(__file__)) print("Using verify-commits data from " + dirname) - verified_root = open(dirname + "/trusted-git-root", "r").read().splitlines()[0] - verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r").read().splitlines()[0] - revsig_allowed = open(dirname + "/allow-revsig-commits", "r").read().splitlines() - unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r").read().splitlines() - incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r").read().splitlines() + verified_root = open(dirname + "/trusted-git-root", "r", encoding="utf8").read().splitlines()[0] + verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8").read().splitlines()[0] + revsig_allowed = open(dirname + "/allow-revsig-commits", "r", encoding="utf-8").read().splitlines() + unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf-8").read().splitlines() + incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf-8").read().splitlines() # Set commit and branch and set variables current_commit = args.commit diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh index e0266bf08a..42d9ffb752 100755 --- a/contrib/verifybinaries/verify.sh +++ b/contrib/verifybinaries/verify.sh @@ -11,6 +11,7 @@ ### The script returns 0 if everything passes the checks. It returns 1 if either the ### signature check or the hash check doesn't pass. If an error occurs the return value is 2 +export LC_ALL=C function clean_up { for file in $* do diff --git a/contrib/windeploy/detached-sig-create.sh b/contrib/windeploy/detached-sig-create.sh index bf4978d143..15f8108cf0 100755 --- a/contrib/windeploy/detached-sig-create.sh +++ b/contrib/windeploy/detached-sig-create.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C if [ -z "$OSSLSIGNCODE" ]; then OSSLSIGNCODE=osslsigncode fi diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index c2e4e36dff..48746ce0c2 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -17,7 +17,7 @@ pkg install autoconf automake boost-libs git gmake libevent libtool openssl pkgc For the wallet (optional): ``` ./contrib/install_db4.sh `pwd` -export BDB_PREFIX='$PWD/db4' +export BDB_PREFIX="$PWD/db4" ``` See [dependencies.md](dependencies.md) for a complete overview. diff --git a/doc/build-unix.md b/doc/build-unix.md index 25b6dd916e..e884c0ab67 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -70,19 +70,7 @@ tuned to conserve memory with additional CXXFLAGS: Build requirements: - sudo apt-get install build-essential libtool autotools-dev automake pkg-config libssl-dev libevent-dev bsdmainutils python3 - -Options when installing required Boost library files: - -1. On at least Ubuntu 14.04+ and Debian 7+ there are generic names for the -individual boost development packages, so the following can be used to only -install necessary parts of boost: - - sudo apt-get install libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev - -2. If that doesn't work, you can install all boost development packages with: - - sudo apt-get install libboost-all-dev + sudo apt-get install build-essential libtool autotools-dev automake pkg-config libssl-dev libevent-dev bsdmainutils python3 libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev BerkeleyDB is required for the wallet. diff --git a/doc/release-notes-pr10740.md b/doc/release-notes-pr10740.md index c81ea6a4db..a2eb8cd837 100644 --- a/doc/release-notes-pr10740.md +++ b/doc/release-notes-pr10740.md @@ -1,9 +1,10 @@ Dynamic loading and creation of wallets --------------------------------------- -Previously, wallets could only be loaded or created at startup, by specifying `-wallet` parameters on the command line or in the bitcoin.conf file. It is now possible to load and create wallets dynamically at runtime: +Previously, wallets could only be loaded or created at startup, by specifying `-wallet` parameters on the command line or in the bitcoin.conf file. It is now possible to load, create and unload wallets dynamically at runtime: - Existing wallets can be loaded by calling the `loadwallet` RPC. The wallet can be specified as file/directory basename (which must be located in the `walletdir` directory), or as an absolute path to a file/directory. - New wallets can be created (and loaded) by calling the `createwallet` RPC. The provided name must not match a wallet file in the `walletdir` directory or the name of a wallet that is currently loaded. +- Loaded wallets can be unloaded by calling the `unloadwallet` RPC. This feature is currently only available through the RPC interface. diff --git a/doc/release-notes/release-notes-0.16.1.md b/doc/release-notes/release-notes-0.16.1.md new file mode 100644 index 0000000000..d99361ae1d --- /dev/null +++ b/doc/release-notes/release-notes-0.16.1.md @@ -0,0 +1,145 @@ +Bitcoin Core version 0.16.1 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-0.16.1/> + +This is a new minor version release, with various bugfixes +as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes for older versions), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +The first time you run version 0.15.0 or newer, your chainstate database will be converted to a +new format, which will take anywhere from a few minutes to half an hour, +depending on the speed of your machine. + +Note that the block database format also changed in version 0.8.0 and there is no +automatic upgrade code from before version 0.8 to version 0.15.0 or higher. Upgrading +directly from 0.7.x and earlier without re-downloading the blockchain is not supported. +However, as usual, old wallet versions are still supported. + +Downgrading warning +------------------- + +Wallets created in 0.16 and later are not compatible with versions prior to 0.16 +and will not work if you try to use newly created wallets in older versions. Existing +wallets that were created with older versions are not affected by this. + +Compatibility +============== + +Bitcoin Core is extensively tested on multiple operating systems using +the Linux kernel, macOS 10.8+, and Windows Vista and later. Windows XP is not supported. + +Bitcoin Core should also work on most other Unix-like systems but is not +frequently tested on them. + +Notable changes +=============== + +Miner block size removed +------------------------ + +The `-blockmaxsize` option for miners to limit their blocks' sizes was +deprecated in version 0.15.1, and has now been removed. Miners should use the +`-blockmaxweight` option if they want to limit the weight of their blocks' +weights. + +0.16.1 change log +------------------ + +### Policy +- #11423 `d353dd1` [Policy] Several transaction standardness rules (jl2012) + +### Mining +- #12756 `e802c22` [config] Remove blockmaxsize option (jnewbery) + +### Block and transaction handling +- #13199 `c71e535` Bugfix: ensure consistency of m_failed_blocks after reconsiderblock (sdaftuar) +- #13023 `bb79aaf` Fix some concurrency issues in ActivateBestChain() (skeees) + +### P2P protocol and network code +- #12626 `f60e84d` Limit the number of IPs addrman learns from each DNS seeder (EthanHeilman) + +### Wallet +- #13265 `5d8de76` Exit SyncMetaData if there are no transactions to sync (laanwj) +- #13030 `5ff571e` Fix zapwallettxes/multiwallet interaction. (jnewbery) + +### GUI +- #12999 `1720eb3` Show the Window when double clicking the taskbar icon (ken2812221) +- #12650 `f118a7a` Fix issue: "default port not shown correctly in settings dialog" (251Labs) +- #13251 `ea487f9` Rephrase Bech32 checkbox texts, and enable it with legacy address default (fanquake) + +### Build system +- #12474 `b0f692f` Allow depends system to support armv7l (hkjn) +- #12585 `72a3290` depends: Switch to downloading expat from GitHub (fanquake) +- #12648 `46ca8f3` test: Update trusted git root (MarcoFalke) +- #11995 `686cb86` depends: Fix Qt build with Xcode 9 (fanquake) +- #12636 `845838c` backport: #11995 Fix Qt build with Xcode 9 (fanquake) +- #12946 `e055bc0` depends: Fix Qt build with XCode 9.3 (fanquake) +- #12998 `7847b92` Default to defining endian-conversion DECLs in compat w/o config (TheBlueMatt) + +### Tests and QA +- #12447 `01f931b` Add missing signal.h header (laanwj) +- #12545 `1286f3e` Use wait_until to ensure ping goes out (Empact) +- #12804 `4bdb0ce` Fix intermittent rpc_net.py failure. (jnewbery) +- #12553 `0e98f96` Prefer wait_until over polling with time.sleep (Empact) +- #12486 `cfebd40` Round target fee to 8 decimals in assert_fee_amount (kallewoof) +- #12843 `df38b13` Test starting bitcoind with -h and -version (jnewbery) +- #12475 `41c29f6` Fix python TypeError in script.py (MarcoFalke) +- #12638 `0a76ed2` Cache only chain and wallet for regtest datadir (MarcoFalke) +- #12902 `7460945` Handle potential cookie race when starting node (sdaftuar) +- #12904 `6c26df0` Ensure bitcoind processes are cleaned up when tests end (sdaftuar) +- #13049 `9ea62a3` Backports (MarcoFalke) +- #13201 `b8aacd6` Handle disconnect_node race (sdaftuar) + +### Miscellaneous +- #12518 `a17fecf` Bump leveldb subtree (MarcoFalke) +- #12442 `f3b8d85` devtools: Exclude patches from lint-whitespace (MarcoFalke) +- #12988 `acdf433` Hold cs_main while calling UpdatedBlockTip() signal (skeees) +- #12985 `0684cf9` Windows: Avoid launching as admin when NSIS installer ends. (JeremyRand) + +### Documentation +- #12637 `60086dd` backport: #12556 fix version typo in getpeerinfo RPC call help (fanquake) +- #13184 `4087dd0` RPC Docs: `gettxout*`: clarify bestblock and unspent counts (harding) +- #13246 `6de7543` Bump to Ubuntu Bionic 18.04 in build-windows.md (ken2812221) +- #12556 `e730b82` Fix version typo in getpeerinfo RPC call help (tamasblummer) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 251 +- Ben Woosley +- Chun Kuan Lee +- David A. Harding +- e0 +- fanquake +- Henrik Jonsson +- JeremyRand +- Jesse Cohen +- John Newbery +- Johnson Lau +- Karl-Johan Alm +- Luke Dashjr +- MarcoFalke +- Matt Corallo +- Pieter Wuille +- Suhas Daftuar +- Tamas Blummer +- Wladimir J. van der Laan + +As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). diff --git a/share/genbuild.sh b/share/genbuild.sh index 419e0da0fd..38c9ba176c 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C if [ $# -gt 1 ]; then cd "$2" || exit 1 fi diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index e8f0820ca8..e908071321 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -63,7 +63,7 @@ child = Popen([XGETTEXT,'--output=-','-n','--keyword=_'] + files, stdout=PIPE) messages = parse_po(out.decode('utf-8')) -f = open(OUT_CPP, 'w') +f = open(OUT_CPP, 'w', encoding="utf8") f.write(""" #include <QtGlobal> diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 2a594c3051..e6eb723cf4 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -591,7 +591,7 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (!prevOut.checkObject(types)) throw std::runtime_error("prevtxs internal object typecheck fail"); - uint256 txid = ParseHashUV(prevOut["txid"], "txid"); + uint256 txid = ParseHashStr(prevOut["txid"].get_str(), "txid"); int nOut = atoi(prevOut["vout"].getValStr()); if (nOut < 0) diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index a9b952e5a4..4b9abb2a1b 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -62,9 +62,6 @@ static bool AppInit(int argc, char* argv[]) // // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() SetupServerArgs(); -#if HAVE_DECL_DAEMON - gArgs.AddArg("-daemon", "Run in the background as a daemon and accept commands", false, OptionsCategory::OPTIONS); -#endif std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { fprintf(stderr, "Error parsing command line arguments: %s\n", error.c_str()); diff --git a/src/core_io.h b/src/core_io.h index 377633ac77..1d87d21d40 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -22,7 +22,6 @@ CScript ParseScript(const std::string& s); std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false); bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no_witness = false, bool try_witness = true); bool DecodeHexBlk(CBlock&, const std::string& strHexBlk); -uint256 ParseHashUV(const UniValue& v, const std::string& strName); uint256 ParseHashStr(const std::string&, const std::string& strName); std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strName); diff --git a/src/core_read.cpp b/src/core_read.cpp index aade7e21ca..4d851610ef 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -160,14 +160,6 @@ bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) return true; } -uint256 ParseHashUV(const UniValue& v, const std::string& strName) -{ - std::string strHex; - if (v.isStr()) - strHex = v.getValStr(); - return ParseHashStr(strHex, strName); // Note: ParseHashStr("") throws a runtime_error -} - uint256 ParseHashStr(const std::string& strHex, const std::string& strName) { if (!IsHex(strHex)) // Note: IsHex("") is false diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 51824fbe9f..e62ddc125e 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -446,44 +446,104 @@ void TransformD64Wrapper(unsigned char* out, const unsigned char* in) WriteBE32(out + 28, s[7]); } -bool SelfTest(TransformType tr) { - static const unsigned char in1[65] = {0, 0x80}; - static const unsigned char in2[129] = { - 0, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0 - }; - static const uint32_t init[8] = {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul}; - static const uint32_t out1[8] = {0xe3b0c442ul, 0x98fc1c14ul, 0x9afbf4c8ul, 0x996fb924ul, 0x27ae41e4ul, 0x649b934cul, 0xa495991bul, 0x7852b855ul}; - static const uint32_t out2[8] = {0xce4153b0ul, 0x147c2a86ul, 0x3ed4298eul, 0xe0676bc8ul, 0x79fc77a1ul, 0x2abe1f49ul, 0xb2b055dful, 0x1069523eul}; - uint32_t buf[8]; - memcpy(buf, init, sizeof(buf)); - // Process nothing, and check we remain in the initial state. - tr(buf, nullptr, 0); - if (memcmp(buf, init, sizeof(buf))) return false; - // Process the padded empty string (unaligned) - tr(buf, in1 + 1, 1); - if (memcmp(buf, out1, sizeof(buf))) return false; - // Process 64 spaces (unaligned) - memcpy(buf, init, sizeof(buf)); - tr(buf, in2 + 1, 2); - if (memcmp(buf, out2, sizeof(buf))) return false; - return true; -} - TransformType Transform = sha256::Transform; TransformD64Type TransformD64 = sha256::TransformD64; TransformD64Type TransformD64_4way = nullptr; TransformD64Type TransformD64_8way = nullptr; +bool SelfTest() { + // Input state (equal to the initial SHA256 state) + static const uint32_t init[8] = { + 0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul + }; + // Some random input data to test with + static const unsigned char data[641] = "-" // Intentionally not aligned + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua. Et m" + "olestie ac feugiat sed lectus vestibulum mattis ullamcorper. Mor" + "bi blandit cursus risus at ultrices mi tempus imperdiet nulla. N" + "unc congue nisi vita suscipit tellus mauris. Imperdiet proin fer" + "mentum leo vel orci. Massa tempor nec feugiat nisl pretium fusce" + " id velit. Telus in metus vulputate eu scelerisque felis. Mi tem" + "pus imperdiet nulla malesuada pellentesque. Tristique magna sit."; + // Expected output state for hashing the i*64 first input bytes above (excluding SHA256 padding). + static const uint32_t result[9][8] = { + {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul}, + {0x91f8ec6bul, 0x4da10fe3ul, 0x1c9c292cul, 0x45e18185ul, 0x435cc111ul, 0x3ca26f09ul, 0xeb954caeul, 0x402a7069ul}, + {0xcabea5acul, 0x374fb97cul, 0x182ad996ul, 0x7bd69cbful, 0x450ff900ul, 0xc1d2be8aul, 0x6a41d505ul, 0xe6212dc3ul}, + {0xbcff09d6ul, 0x3e76f36eul, 0x3ecb2501ul, 0x78866e97ul, 0xe1c1e2fdul, 0x32f4eafful, 0x8aa6c4e5ul, 0xdfc024bcul}, + {0xa08c5d94ul, 0x0a862f93ul, 0x6b7f2f40ul, 0x8f9fae76ul, 0x6d40439ful, 0x79dcee0cul, 0x3e39ff3aul, 0xdc3bdbb1ul}, + {0x216a0895ul, 0x9f1a3662ul, 0xe99946f9ul, 0x87ba4364ul, 0x0fb5db2cul, 0x12bed3d3ul, 0x6689c0c7ul, 0x292f1b04ul}, + {0xca3067f8ul, 0xbc8c2656ul, 0x37cb7e0dul, 0x9b6b8b0ful, 0x46dc380bul, 0xf1287f57ul, 0xc42e4b23ul, 0x3fefe94dul}, + {0x3e4c4039ul, 0xbb6fca8cul, 0x6f27d2f7ul, 0x301e44a4ul, 0x8352ba14ul, 0x5769ce37ul, 0x48a1155ful, 0xc0e1c4c6ul}, + {0xfe2fa9ddul, 0x69d0862bul, 0x1ae0db23ul, 0x471f9244ul, 0xf55c0145ul, 0xc30f9c3bul, 0x40a84ea0ul, 0x5b8a266cul}, + }; + // Expected output for each of the individual 8 64-byte messages under full double SHA256 (including padding). + static const unsigned char result_d64[256] = { + 0x09, 0x3a, 0xc4, 0xd0, 0x0f, 0xf7, 0x57, 0xe1, 0x72, 0x85, 0x79, 0x42, 0xfe, 0xe7, 0xe0, 0xa0, + 0xfc, 0x52, 0xd7, 0xdb, 0x07, 0x63, 0x45, 0xfb, 0x53, 0x14, 0x7d, 0x17, 0x22, 0x86, 0xf0, 0x52, + 0x48, 0xb6, 0x11, 0x9e, 0x6e, 0x48, 0x81, 0x6d, 0xcc, 0x57, 0x1f, 0xb2, 0x97, 0xa8, 0xd5, 0x25, + 0x9b, 0x82, 0xaa, 0x89, 0xe2, 0xfd, 0x2d, 0x56, 0xe8, 0x28, 0x83, 0x0b, 0xe2, 0xfa, 0x53, 0xb7, + 0xd6, 0x6b, 0x07, 0x85, 0x83, 0xb0, 0x10, 0xa2, 0xf5, 0x51, 0x3c, 0xf9, 0x60, 0x03, 0xab, 0x45, + 0x6c, 0x15, 0x6e, 0xef, 0xb5, 0xac, 0x3e, 0x6c, 0xdf, 0xb4, 0x92, 0x22, 0x2d, 0xce, 0xbf, 0x3e, + 0xe9, 0xe5, 0xf6, 0x29, 0x0e, 0x01, 0x4f, 0xd2, 0xd4, 0x45, 0x65, 0xb3, 0xbb, 0xf2, 0x4c, 0x16, + 0x37, 0x50, 0x3c, 0x6e, 0x49, 0x8c, 0x5a, 0x89, 0x2b, 0x1b, 0xab, 0xc4, 0x37, 0xd1, 0x46, 0xe9, + 0x3d, 0x0e, 0x85, 0xa2, 0x50, 0x73, 0xa1, 0x5e, 0x54, 0x37, 0xd7, 0x94, 0x17, 0x56, 0xc2, 0xd8, + 0xe5, 0x9f, 0xed, 0x4e, 0xae, 0x15, 0x42, 0x06, 0x0d, 0x74, 0x74, 0x5e, 0x24, 0x30, 0xce, 0xd1, + 0x9e, 0x50, 0xa3, 0x9a, 0xb8, 0xf0, 0x4a, 0x57, 0x69, 0x78, 0x67, 0x12, 0x84, 0x58, 0xbe, 0xc7, + 0x36, 0xaa, 0xee, 0x7c, 0x64, 0xa3, 0x76, 0xec, 0xff, 0x55, 0x41, 0x00, 0x2a, 0x44, 0x68, 0x4d, + 0xb6, 0x53, 0x9e, 0x1c, 0x95, 0xb7, 0xca, 0xdc, 0x7f, 0x7d, 0x74, 0x27, 0x5c, 0x8e, 0xa6, 0x84, + 0xb5, 0xac, 0x87, 0xa9, 0xf3, 0xff, 0x75, 0xf2, 0x34, 0xcd, 0x1a, 0x3b, 0x82, 0x2c, 0x2b, 0x4e, + 0x6a, 0x46, 0x30, 0xa6, 0x89, 0x86, 0x23, 0xac, 0xf8, 0xa5, 0x15, 0xe9, 0x0a, 0xaa, 0x1e, 0x9a, + 0xd7, 0x93, 0x6b, 0x28, 0xe4, 0x3b, 0xfd, 0x59, 0xc6, 0xed, 0x7c, 0x5f, 0xa5, 0x41, 0xcb, 0x51 + }; + + + // Test Transform() for 0 through 8 transformations. + for (size_t i = 0; i <= 8; ++i) { + uint32_t state[8]; + std::copy(init, init + 8, state); + Transform(state, data + 1, i); + if (!std::equal(state, state + 8, result[i])) return false; + } + + // Test TransformD64 + unsigned char out[32]; + TransformD64(out, data + 1); + if (!std::equal(out, out + 32, result_d64)) return false; + + // Test TransformD64_4way, if available. + if (TransformD64_4way) { + unsigned char out[128]; + TransformD64_4way(out, data + 1); + if (!std::equal(out, out + 128, result_d64)) return false; + } + + // Test TransformD64_8way, if available. + if (TransformD64_8way) { + unsigned char out[256]; + TransformD64_8way(out, data + 1); + if (!std::equal(out, out + 256, result_d64)) return false; + } + + return true; +} + + #if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) // We can't use cpuid.h's __get_cpuid as it does not support subleafs. void inline cpuid(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) { __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); } + +/** Check whether the OS has enabled AVX registers. */ +bool AVXEnabled() +{ + uint32_t a, d; + __asm__("xgetbv" : "=a"(a), "=d"(d) : "c"(0)); + return (a & 6) == 6; +} #endif } // namespace @@ -492,6 +552,7 @@ std::string SHA256AutoDetect() { std::string ret = "standard"; #if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) + (void)AVXEnabled; // Silence unused warning (in case ENABLE_AVX2 is not defined) uint32_t eax, ebx, ecx, edx; cpuid(1, 0, eax, ebx, ecx, edx); if ((ecx >> 19) & 1) { @@ -503,10 +564,14 @@ std::string SHA256AutoDetect() TransformD64_4way = sha256d64_sse41::Transform_4way; ret = "sse4(1way+4way)"; #if defined(ENABLE_AVX2) && !defined(BUILD_BITCOIN_INTERNAL) - cpuid(7, 0, eax, ebx, ecx, edx); - if ((ebx >> 5) & 1) { - TransformD64_8way = sha256d64_avx2::Transform_8way; - ret += ",avx2(8way)"; + if (((ecx >> 27) & 1) && ((ecx >> 28) & 1)) { // XSAVE and AVX + cpuid(7, 0, eax, ebx, ecx, edx); + if ((ebx >> 5) & 1) { // AVX2 flag + if (AVXEnabled()) { // OS has enabled AVX registers + TransformD64_8way = sha256d64_avx2::Transform_8way; + ret += ",avx2(8way)"; + } + } } #endif #else @@ -515,7 +580,7 @@ std::string SHA256AutoDetect() } #endif - assert(SelfTest(Transform)); + assert(SelfTest()); return ret; } diff --git a/src/init.cpp b/src/init.cpp index 9246f6e71c..1b5507703c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -76,7 +76,7 @@ std::unique_ptr<PeerLogicValidation> peerLogic; class DummyWalletInit : public WalletInitInterface { public: - void AddWalletOptions() const override {} + void AddWalletOptions() const override; bool ParameterInteraction() const override {return true;} void RegisterRPC(CRPCTable &) const override {} bool Verify() const override {return true;} @@ -87,6 +87,15 @@ public: void Close() const override {} }; +void DummyWalletInit::AddWalletOptions() const +{ + std::vector<std::string> opts = {"-addresstype", "-changetype", "-disablewallet", "-discardfee=<amt>", "-fallbackfee=<amt>", + "-keypool=<n>", "-mintxfee=<amt>", "-paytxfee=<amt>", "-rescan", "-salvagewallet", "-spendzeroconfchange", "-txconfirmtarget=<n>", + "-upgradewallet", "-wallet=<path>", "-walletbroadcast", "-walletdir=<dir>", "-walletnotify=<cmd>", "-walletrbf", "-zapwallettxes=<mode>", + "-dblogsize=<n>", "-flushwallet", "-privdb", "-walletrejectlongchains"}; + gArgs.AddHiddenArgs(opts); +} + const WalletInitInterface& g_wallet_init_interface = DummyWalletInit(); #endif @@ -348,6 +357,12 @@ void SetupServerArgs() const auto defaultChainParams = CreateChainParams(CBaseChainParams::MAIN); const auto testnetChainParams = CreateChainParams(CBaseChainParams::TESTNET); + // Hidden Options + std::vector<std::string> hidden_args = {"-rpcssl", "-benchmark", "-h", "-help", "-socks", "-tor", "-debugnet", "-whitelistalwaysrelay", + "-prematurewitness", "-walletprematurewitness", "-promiscuousmempoolflags", "-blockminsize", "-dbcrashratio", "-forcecompactdb", "-usehd", + // GUI args. These will be overwritten by SetupUIArgs for the GUI + "-allowselfsignedrootcertificates", "-choosedatadir", "-lang=<lang>", "-min", "-resetguisettings", "-rootcertificates=<file>", "-splash", "-uiplatform"}; + // Set all of the args and their help // When adding new options to the categories, please keep and ensure alphabetical ordering. gArgs.AddArg("-?", "Print this help message and exit", false, OptionsCategory::OPTIONS); @@ -375,6 +390,8 @@ void SetupServerArgs() gArgs.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), false, OptionsCategory::OPTIONS); #ifndef WIN32 gArgs.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), false, OptionsCategory::OPTIONS); +#else + hidden_args.emplace_back("-pid"); #endif gArgs.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " @@ -383,6 +400,8 @@ void SetupServerArgs() gArgs.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks", false, OptionsCategory::OPTIONS); #ifndef WIN32 gArgs.AddArg("-sysperms", "Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)", false, OptionsCategory::OPTIONS); +#else + hidden_args.emplace_back("-sysperms"); #endif gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS); @@ -421,6 +440,8 @@ void SetupServerArgs() #else gArgs.AddArg("-upnp", strprintf("Use UPnP to map the listening port (default: %u)", 0), false, OptionsCategory::CONNECTION); #endif +#else + hidden_args.emplace_back("-upnp"); #endif gArgs.AddArg("-whitebind=<addr>", "Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6", false, OptionsCategory::CONNECTION); gArgs.AddArg("-whitelist=<IP address or network>", "Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or CIDR notated network (e.g. 1.2.3.0/24). Can be specified multiple times." @@ -433,6 +454,11 @@ void SetupServerArgs() gArgs.AddArg("-zmqpubhashtx=<address>", "Enable publish hash transaction in <address>", false, OptionsCategory::ZMQ); gArgs.AddArg("-zmqpubrawblock=<address>", "Enable publish raw block in <address>", false, OptionsCategory::ZMQ); gArgs.AddArg("-zmqpubrawtx=<address>", "Enable publish raw transaction in <address>", false, OptionsCategory::ZMQ); +#else + hidden_args.emplace_back("-zmqpubhashblock=<address>"); + hidden_args.emplace_back("-zmqpubhashtx=<address>"); + hidden_args.emplace_back("-zmqpubrawblock=<address>"); + hidden_args.emplace_back("-zmqpubrawtx=<address>"); #endif gArgs.AddArg("-checkblocks=<n>", strprintf("How many blocks to check at startup (default: %u, 0 = all)", DEFAULT_CHECKBLOCKS), true, OptionsCategory::DEBUG_TEST); @@ -500,22 +526,14 @@ void SetupServerArgs() gArgs.AddArg("-rpcworkqueue=<n>", strprintf("Set the depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE), true, OptionsCategory::RPC); gArgs.AddArg("-server", "Accept command line and JSON-RPC commands", false, OptionsCategory::RPC); - // Hidden options - gArgs.AddArg("-rpcssl", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-benchmark", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-h", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-help", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-socks", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-tor", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-debugnet", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-whitelistalwaysrelay", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-prematurewitness", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-walletprematurewitness", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-promiscuousmempoolflags", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-blockminsize", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-dbcrashratio", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-forcecompactdb", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-usehd", "", false, OptionsCategory::HIDDEN); +#if HAVE_DECL_DAEMON + gArgs.AddArg("-daemon", "Run in the background as a daemon and accept commands", false, OptionsCategory::OPTIONS); +#else + hidden_args.emplace_back("-daemon"); +#endif + + // Add the hidden options + gArgs.AddHiddenArgs(hidden_args); } std::string LicenseInfo() @@ -614,7 +632,7 @@ static void CleanupBlockRevFiles() // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) // start removing block files. int nContigCounter = 0; - for (const std::pair<std::string, fs::path>& item : mapBlockFiles) { + for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) { if (atoi(item.first) == nContigCounter) { nContigCounter++; continue; diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 3029dbe8e3..e98acba0df 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -429,6 +429,10 @@ public: bool hdEnabled() override { return m_wallet.IsHDEnabled(); } OutputType getDefaultAddressType() override { return m_wallet.m_default_address_type; } OutputType getDefaultChangeType() override { return m_wallet.m_default_change_type; } + std::unique_ptr<Handler> handleUnload(UnloadFn fn) override + { + return MakeHandler(m_wallet.NotifyUnload.connect(fn)); + } std::unique_ptr<Handler> handleShowProgress(ShowProgressFn fn) override { return MakeHandler(m_wallet.ShowProgress.connect(fn)); diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 82ae0b14b5..ce42e14eea 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -242,6 +242,10 @@ public: // Get default change type. virtual OutputType getDefaultChangeType() = 0; + //! Register handler for unload message. + using UnloadFn = std::function<void()>; + virtual std::unique_ptr<Handler> handleUnload(UnloadFn fn) = 0; + //! Register handler for show progress messages. using ShowProgressFn = std::function<void(const std::string& title, int progress)>; virtual std::unique_ptr<Handler> handleShowProgress(ShowProgressFn fn) = 0; diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 1c846d38ec..360615ec56 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -388,11 +388,6 @@ struct CMutableTransaction */ uint256 GetHash() const; - friend bool operator==(const CMutableTransaction& a, const CMutableTransaction& b) - { - return a.GetHash() == b.GetHash(); - } - bool HasWitness() const { for (size_t i = 0; i < vin.size(); i++) { diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index d38175e4a5..e3d1c746b1 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -215,6 +215,7 @@ public Q_SLOTS: /// Handle runaway exceptions. Shows a message box with the problem and quits the program. void handleRunawayException(const QString &message); void addWallet(WalletModel* walletModel); + void removeWallet(); Q_SIGNALS: void requestedInitialize(); @@ -444,11 +445,22 @@ void BitcoinApplication::addWallet(WalletModel* walletModel) connect(walletModel, SIGNAL(coinsSent(WalletModel*, SendCoinsRecipient, QByteArray)), paymentServer, SLOT(fetchPaymentACK(WalletModel*, const SendCoinsRecipient&, QByteArray))); + connect(walletModel, SIGNAL(unload()), this, SLOT(removeWallet())); m_wallet_models.push_back(walletModel); #endif } +void BitcoinApplication::removeWallet() +{ +#ifdef ENABLE_WALLET + WalletModel* walletModel = static_cast<WalletModel*>(sender()); + m_wallet_models.erase(std::find(m_wallet_models.begin(), m_wallet_models.end(), walletModel)); + window->removeWallet(walletModel); + walletModel->deleteLater(); +#endif +} + void BitcoinApplication::initializeResult(bool success) { qDebug() << __func__ << ": Initialization result: " << success; @@ -468,8 +480,10 @@ void BitcoinApplication::initializeResult(bool success) #ifdef ENABLE_WALLET m_handler_load_wallet = m_node.handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { - QMetaObject::invokeMethod(this, "addWallet", Qt::QueuedConnection, - Q_ARG(WalletModel*, new WalletModel(std::move(wallet), m_node, platformStyle, optionsModel))); + WalletModel* wallet_model = new WalletModel(std::move(wallet), m_node, platformStyle, optionsModel, nullptr); + // Fix wallet model thread affinity. + wallet_model->moveToThread(thread()); + QMetaObject::invokeMethod(this, "addWallet", Qt::QueuedConnection, Q_ARG(WalletModel*, wallet_model)); }); for (auto& wallet : m_node.getWallets()) { diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 0677a2422c..b3c8f57deb 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -114,6 +114,7 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty modalOverlay(0), prevBlocks(0), spinnerFrame(0), + m_wallet_selector_label(nullptr), platformStyle(_platformStyle) { QSettings settings; @@ -465,6 +466,16 @@ void BitcoinGUI::createToolBars() m_wallet_selector = new QComboBox(); connect(m_wallet_selector, SIGNAL(currentIndexChanged(int)), this, SLOT(setCurrentWalletBySelectorIndex(int))); + + m_wallet_selector_label = new QLabel(); + m_wallet_selector_label->setText(tr("Wallet:") + " "); + m_wallet_selector_label->setBuddy(m_wallet_selector); + + m_wallet_selector_label_action = appToolBar->addWidget(m_wallet_selector_label); + m_wallet_selector_action = appToolBar->addWidget(m_wallet_selector); + + m_wallet_selector_label_action->setVisible(false); + m_wallet_selector_action->setVisible(false); #endif } } @@ -544,16 +555,29 @@ bool BitcoinGUI::addWallet(WalletModel *walletModel) setWalletActionsEnabled(true); m_wallet_selector->addItem(display_name, name); if (m_wallet_selector->count() == 2) { - m_wallet_selector_label = new QLabel(); - m_wallet_selector_label->setText(tr("Wallet:") + " "); - m_wallet_selector_label->setBuddy(m_wallet_selector); - appToolBar->addWidget(m_wallet_selector_label); - appToolBar->addWidget(m_wallet_selector); + m_wallet_selector_label_action->setVisible(true); + m_wallet_selector_action->setVisible(true); } rpcConsole->addWallet(walletModel); return walletFrame->addWallet(walletModel); } +bool BitcoinGUI::removeWallet(WalletModel* walletModel) +{ + if (!walletFrame) return false; + QString name = walletModel->getWalletName(); + int index = m_wallet_selector->findData(name); + m_wallet_selector->removeItem(index); + if (m_wallet_selector->count() == 0) { + setWalletActionsEnabled(false); + } else if (m_wallet_selector->count() == 1) { + m_wallet_selector_label_action->setVisible(false); + m_wallet_selector_action->setVisible(false); + } + rpcConsole->removeWallet(walletModel); + return walletFrame->removeWallet(name); +} + bool BitcoinGUI::setCurrentWallet(const QString& name) { if(!walletFrame) diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 89c1c73a79..68c35557cc 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -70,6 +70,7 @@ public: functionality. */ bool addWallet(WalletModel *walletModel); + bool removeWallet(WalletModel* walletModel); void removeAllWallets(); #endif // ENABLE_WALLET bool enableWallet; @@ -122,8 +123,10 @@ private: QAction *openRPCConsoleAction; QAction *openAction; QAction *showHelpMessageAction; + QAction *m_wallet_selector_label_action = nullptr; + QAction *m_wallet_selector_action = nullptr; - QLabel *m_wallet_selector_label; + QLabel *m_wallet_selector_label = nullptr; QComboBox *m_wallet_selector; QSystemTrayIcon *trayIcon; diff --git a/src/qt/res/movies/makespinner.sh b/src/qt/res/movies/makespinner.sh index d0deb1238c..76e36e4f31 100755 --- a/src/qt/res/movies/makespinner.sh +++ b/src/qt/res/movies/makespinner.sh @@ -2,6 +2,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C FRAMEDIR=$(dirname $0) for i in {0..35} do diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index b47eb9c02b..f222357f27 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -709,6 +709,16 @@ void RPCConsole::addWallet(WalletModel * const walletModel) ui->WalletSelectorLabel->setVisible(true); } } + +void RPCConsole::removeWallet(WalletModel * const walletModel) +{ + const QString name = walletModel->getWalletName(); + ui->WalletSelector->removeItem(ui->WalletSelector->findData(name)); + if (ui->WalletSelector->count() == 2) { + ui->WalletSelector->setVisible(false); + ui->WalletSelectorLabel->setVisible(false); + } +} #endif static QString categoryClass(int category) diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index a53c4c24f9..0a1a469934 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -48,6 +48,7 @@ public: void setClientModel(ClientModel *model); void addWallet(WalletModel * const walletModel); + void removeWallet(WalletModel* const walletModel); enum MessageClass { MC_ERROR, diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index eb0eba21ef..c5a13f61f4 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -94,6 +94,7 @@ bool WalletFrame::removeWallet(const QString &name) WalletView *walletView = mapWalletViews.take(name); walletStack->removeWidget(walletView); + delete walletView; return true; } diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 3418b1f1a9..389acf0a95 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -364,6 +364,12 @@ bool WalletModel::changePassphrase(const SecureString &oldPass, const SecureStri } // Handlers for core signals +static void NotifyUnload(WalletModel* walletModel) +{ + qDebug() << "NotifyUnload"; + QMetaObject::invokeMethod(walletModel, "unload", Qt::QueuedConnection); +} + static void NotifyKeyStoreStatusChanged(WalletModel *walletmodel) { qDebug() << "NotifyKeyStoreStatusChanged"; @@ -411,6 +417,7 @@ static void NotifyWatchonlyChanged(WalletModel *walletmodel, bool fHaveWatchonly void WalletModel::subscribeToCoreSignals() { // Connect signals to wallet + m_handler_unload = m_wallet->handleUnload(boost::bind(&NotifyUnload, this)); m_handler_status_changed = m_wallet->handleStatusChanged(boost::bind(&NotifyKeyStoreStatusChanged, this)); m_handler_address_book_changed = m_wallet->handleAddressBookChanged(boost::bind(NotifyAddressBookChanged, this, _1, _2, _3, _4, _5)); m_handler_transaction_changed = m_wallet->handleTransactionChanged(boost::bind(NotifyTransactionChanged, this, _1, _2)); @@ -421,6 +428,7 @@ void WalletModel::subscribeToCoreSignals() void WalletModel::unsubscribeFromCoreSignals() { // Disconnect signals from wallet + m_handler_unload->disconnect(); m_handler_status_changed->disconnect(); m_handler_address_book_changed->disconnect(); m_handler_transaction_changed->disconnect(); diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 9173fcae52..35ededb121 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -208,6 +208,7 @@ public: AddressTableModel* getAddressTableModel() const { return addressTableModel; } private: std::unique_ptr<interfaces::Wallet> m_wallet; + std::unique_ptr<interfaces::Handler> m_handler_unload; std::unique_ptr<interfaces::Handler> m_handler_status_changed; std::unique_ptr<interfaces::Handler> m_handler_address_book_changed; std::unique_ptr<interfaces::Handler> m_handler_transaction_changed; @@ -261,6 +262,9 @@ Q_SIGNALS: // Watch-only address added void notifyWatchonlyChanged(bool fHaveWatchonly); + // Signal that wallet is about to be removed + void unload(); + public Q_SLOTS: /* Wallet status might have changed */ void updateStatus(); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f70d506e13..68279d7ffd 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -95,6 +95,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex) result.pushKV("bits", strprintf("%08x", blockindex->nBits)); result.pushKV("difficulty", GetDifficulty(blockindex)); result.pushKV("chainwork", blockindex->nChainWork.GetHex()); + result.pushKV("nTx", (uint64_t)blockindex->nTx); if (blockindex->pprev) result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); @@ -140,6 +141,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool tx result.pushKV("bits", strprintf("%08x", block.nBits)); result.pushKV("difficulty", GetDifficulty(blockindex)); result.pushKV("chainwork", blockindex->nChainWork.GetHex()); + result.pushKV("nTx", (uint64_t)blockindex->nTx); if (blockindex->pprev) result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); @@ -694,6 +696,7 @@ static UniValue getblockheader(const JSONRPCRequest& request) " \"bits\" : \"1d00ffff\", (string) The bits\n" " \"difficulty\" : x.xxx, (numeric) The difficulty\n" " \"chainwork\" : \"0000...1f3\" (string) Expected number of hashes required to produce the current chain (in hex)\n" + " \"nTx\" : n, (numeric) The number of transactions in the block.\n" " \"previousblockhash\" : \"hash\", (string) The hash of the previous block\n" " \"nextblockhash\" : \"hash\", (string) The hash of the next block\n" "}\n" @@ -782,6 +785,7 @@ static UniValue getblock(const JSONRPCRequest& request) " \"bits\" : \"1d00ffff\", (string) The bits\n" " \"difficulty\" : x.xxx, (numeric) The difficulty\n" " \"chainwork\" : \"xxxx\", (string) Expected number of hashes required to produce the chain up to this block (in hex)\n" + " \"nTx\" : n, (numeric) The number of transactions in the block.\n" " \"previousblockhash\" : \"hash\", (string) The hash of the previous block\n" " \"nextblockhash\" : \"hash\" (string) The hash of the next block\n" "}\n" diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 85b864e6b9..81c4fb040f 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -725,7 +725,6 @@ static UniValue submitblock(const JSONRPCRequest& request) } uint256 hash = block.GetHash(); - bool fBlockPresent = false; { LOCK(cs_main); const CBlockIndex* pindex = LookupBlockIndex(hash); @@ -736,8 +735,6 @@ static UniValue submitblock(const JSONRPCRequest& request) if (pindex->nStatus & BLOCK_FAILED_MASK) { return "duplicate-invalid"; } - // Otherwise, we might only have the header - process the block before returning - fBlockPresent = true; } } @@ -749,13 +746,15 @@ static UniValue submitblock(const JSONRPCRequest& request) } } + bool new_block; submitblock_StateCatcher sc(block.GetHash()); RegisterValidationInterface(&sc); - bool fAccepted = ProcessNewBlock(Params(), blockptr, true, nullptr); + bool accepted = ProcessNewBlock(Params(), blockptr, /* fForceProcessing */ true, /* fNewBlock */ &new_block); UnregisterValidationInterface(&sc); - if (fBlockPresent) { - if (fAccepted && !sc.found) { - return "duplicate-inconclusive"; + if (!new_block) { + if (!accepted) { + // TODO Maybe pass down fNewBlock to AcceptBlockHeader, so it is properly set to true in this case? + return "invalid"; } return "duplicate"; } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 1530d8578b..8fa56e9335 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -475,7 +475,7 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) UniValue localAddresses(UniValue::VARR); { LOCK(cs_mapLocalHost); - for (const std::pair<CNetAddr, LocalServiceInfo> &item : mapLocalHost) + for (const std::pair<const CNetAddr, LocalServiceInfo> &item : mapLocalHost) { UniValue rec(UniValue::VOBJ); rec.pushKV("address", item.first.ToString()); diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 4070642537..0264d29455 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -587,9 +587,6 @@ inline CTransactionRef make_tx(std::vector<CAmount>&& output_values, std::vector return MakeTransactionRef(tx); } -#define MK_OUTPUTS(amounts...) std::vector<CAmount>{amounts} -#define MK_INPUTS(txs...) std::vector<CTransactionRef>{txs} -#define MK_INPUT_IDX(idxes...) std::vector<uint32_t>{idxes} BOOST_AUTO_TEST_CASE(MempoolAncestryTests) { @@ -602,7 +599,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // // [tx1] // - CTransactionRef tx1 = make_tx(MK_OUTPUTS(10 * COIN)); + CTransactionRef tx1 = make_tx(/* output_values */ {10 * COIN}); pool.addUnchecked(tx1->GetHash(), entry.Fee(10000LL).FromTx(tx1)); // Ancestors / descendants should be 1 / 1 (itself / itself) @@ -614,7 +611,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // // [tx1].0 <- [tx2] // - CTransactionRef tx2 = make_tx(MK_OUTPUTS(495 * CENT, 5 * COIN), MK_INPUTS(tx1)); + CTransactionRef tx2 = make_tx(/* output_values */ {495 * CENT, 5 * COIN}, /* inputs */ {tx1}); pool.addUnchecked(tx2->GetHash(), entry.Fee(10000LL).FromTx(tx2)); // Ancestors / descendants should be: @@ -633,7 +630,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // // [tx1].0 <- [tx2].0 <- [tx3] // - CTransactionRef tx3 = make_tx(MK_OUTPUTS(290 * CENT, 200 * CENT), MK_INPUTS(tx2)); + CTransactionRef tx3 = make_tx(/* output_values */ {290 * CENT, 200 * CENT}, /* inputs */ {tx2}); pool.addUnchecked(tx3->GetHash(), entry.Fee(10000LL).FromTx(tx3)); // Ancestors / descendants should be: @@ -658,7 +655,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // | // \---1 <- [tx4] // - CTransactionRef tx4 = make_tx(MK_OUTPUTS(290 * CENT, 250 * CENT), MK_INPUTS(tx2), MK_INPUT_IDX(1)); + CTransactionRef tx4 = make_tx(/* output_values */ {290 * CENT, 250 * CENT}, /* inputs */ {tx2}, /* input_indices */ {1}); pool.addUnchecked(tx4->GetHash(), entry.Fee(10000LL).FromTx(tx4)); // Ancestors / descendants should be: @@ -694,14 +691,14 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) CAmount v = 5 * COIN; for (uint64_t i = 0; i < 5; i++) { CTransactionRef& tyi = *ty[i]; - tyi = make_tx(MK_OUTPUTS(v), i > 0 ? MK_INPUTS(*ty[i-1]) : std::vector<CTransactionRef>()); + tyi = make_tx(/* output_values */ {v}, /* inputs */ i > 0 ? std::vector<CTransactionRef>{*ty[i - 1]} : std::vector<CTransactionRef>{}); v -= 50 * CENT; pool.addUnchecked(tyi->GetHash(), entry.Fee(10000LL).FromTx(tyi)); pool.GetTransactionAncestry(tyi->GetHash(), ancestors, descendants); BOOST_CHECK_EQUAL(ancestors, i+1); BOOST_CHECK_EQUAL(descendants, i+1); } - CTransactionRef ty6 = make_tx(MK_OUTPUTS(5 * COIN), MK_INPUTS(tx3, ty5)); + CTransactionRef ty6 = make_tx(/* output_values */ {5 * COIN}, /* inputs */ {tx3, ty5}); pool.addUnchecked(ty6->GetHash(), entry.Fee(10000LL).FromTx(ty6)); // Ancestors / descendants should be: diff --git a/src/util.cpp b/src/util.cpp index 48d64e3eec..ab262b4063 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -585,6 +585,13 @@ void ArgsManager::AddArg(const std::string& name, const std::string& help, const assert(ret.second); // Make sure an insertion actually happened } +void ArgsManager::AddHiddenArgs(const std::vector<std::string>& names) +{ + for (const std::string& name : names) { + AddArg(name, "", false, OptionsCategory::HIDDEN); + } +} + std::string ArgsManager::GetHelpMessage() { const bool show_debug = gArgs.GetBoolArg("-help-debug", false); diff --git a/src/util.h b/src/util.h index efd8a4bd9d..8094d72d6b 100644 --- a/src/util.h +++ b/src/util.h @@ -264,6 +264,11 @@ public: void AddArg(const std::string& name, const std::string& help, const bool debug_only, const OptionsCategory& cat); /** + * Add many hidden arguments + */ + void AddHiddenArgs(const std::vector<std::string>& args); + + /** * Clear available arguments */ void ClearArgs() { m_available_args.clear(); } diff --git a/src/validation.cpp b/src/validation.cpp index bbf2389d34..baf083b153 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -205,7 +205,7 @@ private: void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state); CBlockIndex* FindMostWorkChain(); - bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams); + void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams); bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params); @@ -2698,6 +2698,9 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& // Block until the validation queue drains. This should largely // never happen in normal operation, however may happen during // reindex, causing memory blowup if we run too far ahead. + // Note that if a validationinterface callback ends up calling + // ActivateBestChain this may lead to a deadlock! We should + // probably have a DEBUG_LOCKORDER test for this in the future. SyncWithValidationInterfaceQueue(); } @@ -2950,7 +2953,7 @@ CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -bool CChainState::ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBlockIndex *pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2994,8 +2997,6 @@ bool CChainState::ReceivedBlockTransactions(const CBlock &block, CValidationStat mapBlocksUnlinked.insert(std::make_pair(pindexNew->pprev, pindexNew)); } } - - return true; } static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) @@ -3507,7 +3508,6 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali // request; don't process these. if (pindex->nChainWork < nMinimumChainWork) return true; } - if (fNewBlock) *fNewBlock = true; if (!CheckBlock(block, state, chainparams.GetConsensus()) || !ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindex->pprev)) { @@ -3524,14 +3524,14 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali GetMainSignals().NewPoWValidBlock(pindex, pblock); // Write block to history file + if (fNewBlock) *fNewBlock = true; try { CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; } - if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) - return error("AcceptBlock(): ReceivedBlockTransactions failed"); + ReceivedBlockTransactions(block, pindex, blockPos, chainparams.GetConsensus()); } catch (const std::runtime_error& e) { return AbortNode(state, std::string("System error: ") + e.what()); } @@ -3834,7 +3834,7 @@ bool CChainState::LoadBlockIndex(const Consensus::Params& consensus_params, CBlo // Calculate nChainWork std::vector<std::pair<int, CBlockIndex*> > vSortedByHeight; vSortedByHeight.reserve(mapBlockIndex.size()); - for (const std::pair<uint256, CBlockIndex*>& item : mapBlockIndex) + for (const std::pair<const uint256, CBlockIndex*>& item : mapBlockIndex) { CBlockIndex* pindex = item.second; vSortedByHeight.push_back(std::make_pair(pindex->nHeight, pindex)); @@ -3901,7 +3901,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - for (const std::pair<uint256, CBlockIndex*>& item : mapBlockIndex) + for (const std::pair<const uint256, CBlockIndex*>& item : mapBlockIndex) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { @@ -4340,9 +4340,7 @@ bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) if (blockPos.IsNull()) return error("%s: writing genesis block to disk failed", __func__); CBlockIndex *pindex = AddToBlockIndex(block); - CValidationState state; - if (!ReceivedBlockTransactions(block, state, pindex, blockPos, chainparams.GetConsensus())) - return error("%s: genesis block not accepted (%s)", __func__, FormatStateMessage(state)); + ReceivedBlockTransactions(block, pindex, blockPos, chainparams.GetConsensus()); } catch (const std::runtime_error& e) { return error("%s: failed to write genesis block: %s", __func__, e.what()); } diff --git a/src/validation.h b/src/validation.h index 04f5b6cb80..b9c2f6c023 100644 --- a/src/validation.h +++ b/src/validation.h @@ -233,7 +233,8 @@ static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; * Note that we guarantee that either the proof-of-work is valid on pblock, or * (and possibly also) BlockChecked will have been called. * - * Call without cs_main held. + * May not be called with cs_main held. May not be called in a + * validationinterface callback. * * @param[in] pblock The block we want to process. * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. @@ -245,7 +246,8 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons /** * Process incoming block headers. * - * Call without cs_main held. + * May not be called with cs_main held. May not be called in a + * validationinterface callback. * * @param[in] block The block headers themselves * @param[out] state This may be set to an Error state if any error occurred processing them @@ -278,7 +280,12 @@ void ThreadScriptCheck(); bool IsInitialBlockDownload(); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ bool GetTransaction(const uint256& hash, CTransactionRef& tx, const Consensus::Params& params, uint256& hashBlock, bool fAllowSlow = false, CBlockIndex* blockIndex = nullptr); -/** Find the best known block, and make it the tip of the block chain */ +/** + * Find the best known block, and make it the tip of the block chain + * + * May not be called with cs_main held. May not be called in a + * validationinterface callback. + */ bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); @@ -445,7 +452,11 @@ inline CBlockIndex* LookupBlockIndex(const uint256& hash) /** Find the last common block between the parameter chain and a locator. */ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator); -/** Mark a block as precious and reorganize. */ +/** Mark a block as precious and reorganize. + * + * May not be called with cs_main held. May not be called in a + * validationinterface callback. + */ bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex); /** Mark a block as invalid. */ diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 410dd5009f..01b8eacccb 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -694,8 +694,10 @@ void BerkeleyEnvironment::Flush(bool fShutdown) if (mapFileUseCount.empty()) { dbenv->log_archive(&listp, DB_ARCH_REMOVE); Close(); - if (!fMockDb) + if (!fMockDb) { fs::remove_all(fs::path(strPath) / "database"); + } + g_dbenvs.erase(strPath); } } } @@ -794,5 +796,6 @@ void BerkeleyDatabase::Flush(bool shutdown) { if (!IsDummy()) { env->Flush(shutdown); + if (shutdown) env = nullptr; } } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 94c35e2cb1..3f2255d042 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -40,12 +40,21 @@ static const std::string WALLET_ENDPOINT_BASE = "/wallet/"; -std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& request) +bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& wallet_name) { if (request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { // wallet endpoint was used - std::string requestedWallet = urlDecode(request.URI.substr(WALLET_ENDPOINT_BASE.size())); - std::shared_ptr<CWallet> pwallet = GetWallet(requestedWallet); + wallet_name = urlDecode(request.URI.substr(WALLET_ENDPOINT_BASE.size())); + return true; + } + return false; +} + +std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& request) +{ + std::string wallet_name; + if (GetWalletNameFromJSONRPCRequest(request, wallet_name)) { + std::shared_ptr<CWallet> pwallet = GetWallet(wallet_name); if (!pwallet) throw JSONRPCError(RPC_WALLET_NOT_FOUND, "Requested wallet does not exist or is not loaded"); return pwallet; } @@ -66,11 +75,6 @@ bool EnsureWalletIsAvailable(CWallet * const pwallet, bool avoidException) if (pwallet) return true; if (avoidException) return false; if (!HasWallets()) { - // Note: It isn't currently possible to trigger this error because - // wallet RPC methods aren't registered unless a wallet is loaded. But - // this error is being kept as a precaution, because it's possible in - // the future that wallet RPC methods might get or remain registered - // when no wallets are loaded. throw JSONRPCError( RPC_METHOD_NOT_FOUND, "Method not found (wallet method is disabled because no wallet is loaded)"); } @@ -120,7 +124,7 @@ static void WalletTxToJSON(const CWalletTx& wtx, UniValue& entry) } entry.pushKV("bip125-replaceable", rbfStatus); - for (const std::pair<std::string, std::string>& item : wtx.mapValue) + for (const std::pair<const std::string, std::string>& item : wtx.mapValue) entry.pushKV(item.first, item.second); } @@ -343,7 +347,7 @@ static UniValue setlabel(const JSONRPCRequest& request) // If so, delete the account record for it. Labels, unlike addresses, can be deleted, // and if we wouldn't do this, the record would stick around forever. bool found_address = false; - for (const std::pair<CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { + for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { if (item.second.name == label) { found_address = true; break; @@ -440,7 +444,7 @@ static UniValue getaddressesbyaccount(const JSONRPCRequest& request) // Find all addresses that have the given account UniValue ret(UniValue::VARR); - for (const std::pair<CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { + for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { const CTxDestination& dest = item.first; const std::string& strName = item.second.name; if (strName == strAccount) { @@ -753,7 +757,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) // Tally CAmount nAmount = 0; - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; @@ -821,7 +825,7 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) // Tally CAmount nAmount = 0; - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; @@ -1527,7 +1531,7 @@ static UniValue ListReceived(CWallet * const pwallet, const UniValue& params, bo // Tally std::map<CTxDestination, tallyitem> mapTally; - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) @@ -2106,13 +2110,13 @@ static UniValue listaccounts(const JSONRPCRequest& request) includeWatchonly = includeWatchonly | ISMINE_WATCH_ONLY; std::map<std::string, CAmount> mapAccountBalances; - for (const std::pair<CTxDestination, CAddressBookData>& entry : pwallet->mapAddressBook) { + for (const std::pair<const CTxDestination, CAddressBookData>& entry : pwallet->mapAddressBook) { if (IsMine(*pwallet, entry.first) & includeWatchonly) { // This address belongs to me mapAccountBalances[entry.second.name] = 0; } } - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; CAmount nFee; std::string strSentAccount; @@ -2141,7 +2145,7 @@ static UniValue listaccounts(const JSONRPCRequest& request) mapAccountBalances[entry.strAccount] += entry.nCreditDebit; UniValue ret(UniValue::VOBJ); - for (const std::pair<std::string, CAmount>& accountBalance : mapAccountBalances) { + for (const std::pair<const std::string, CAmount>& accountBalance : mapAccountBalances) { ret.pushKV(accountBalance.first, ValueFromAmount(accountBalance.second)); } return ret; @@ -2250,7 +2254,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) UniValue transactions(UniValue::VARR); - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { CWalletTx tx = pairWtx.second; if (depth == -1 || tx.GetDepthInMainChain() < depth) { @@ -3053,7 +3057,7 @@ static UniValue listwallets(const JSONRPCRequest& request) return obj; } -UniValue loadwallet(const JSONRPCRequest& request) +static UniValue loadwallet(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) throw std::runtime_error( @@ -3100,7 +3104,7 @@ UniValue loadwallet(const JSONRPCRequest& request) return obj; } -UniValue createwallet(const JSONRPCRequest& request) +static UniValue createwallet(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 1) { throw std::runtime_error( @@ -3147,6 +3151,55 @@ UniValue createwallet(const JSONRPCRequest& request) return obj; } +static UniValue unloadwallet(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() > 1) { + throw std::runtime_error( + "unloadwallet ( \"wallet_name\" )\n" + "Unloads the wallet referenced by the request endpoint otherwise unloads the wallet specified in the argument.\n" + "Specifying the wallet name on a wallet endpoint is invalid." + "\nArguments:\n" + "1. \"wallet_name\" (string, optional) The name of the wallet to unload.\n" + "\nExamples:\n" + + HelpExampleCli("unloadwallet", "wallet_name") + + HelpExampleRpc("unloadwallet", "wallet_name") + ); + } + + std::string wallet_name; + if (GetWalletNameFromJSONRPCRequest(request, wallet_name)) { + if (!request.params[0].isNull()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot unload the requested wallet"); + } + } else { + wallet_name = request.params[0].get_str(); + } + + std::shared_ptr<CWallet> wallet = GetWallet(wallet_name); + if (!wallet) { + throw JSONRPCError(RPC_WALLET_NOT_FOUND, "Requested wallet does not exist or is not loaded"); + } + + // Release the "main" shared pointer and prevent further notifications. + // Note that any attempt to load the same wallet would fail until the wallet + // is destroyed (see CheckUniqueFileid). + if (!RemoveWallet(wallet)) { + throw JSONRPCError(RPC_MISC_ERROR, "Requested wallet already unloaded"); + } + UnregisterValidationInterface(wallet.get()); + + // The wallet can be in use so it's not possible to explicitly unload here. + // Just notify the unload intent so that all shared pointers are released. + // The wallet will be destroyed once the last shared pointer is released. + wallet->NotifyUnload(); + + // There's no point in waiting for the wallet to unload. + // At this point this method should never fail. The unloading could only + // fail due to an unexpected error which would cause a process termination. + + return NullUniValue; +} + static UniValue resendwallettransactions(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -4187,7 +4240,7 @@ static UniValue getaddressesbylabel(const JSONRPCRequest& request) // Find all addresses that have the given label UniValue ret(UniValue::VOBJ); - for (const std::pair<CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { + for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { if (item.second.name == label) { ret.pushKV(EncodeDestination(item.first), AddressBookDataToJSON(item.second, false)); } @@ -4240,7 +4293,7 @@ static UniValue listlabels(const JSONRPCRequest& request) // Add to a set to sort by label name, then insert into Univalue array std::set<std::string> label_set; - for (const std::pair<CTxDestination, CAddressBookData>& entry : pwallet->mapAddressBook) { + for (const std::pair<const CTxDestination, CAddressBookData>& entry : pwallet->mapAddressBook) { if (purpose.empty() || entry.second.purpose == purpose) { label_set.insert(entry.second.name); } @@ -4382,6 +4435,7 @@ static const CRPCCommand commands[] = { "wallet", "settxfee", &settxfee, {"amount"} }, { "wallet", "signmessage", &signmessage, {"address","message"} }, { "wallet", "signrawtransactionwithwallet", &signrawtransactionwithwallet, {"hexstring","prevtxs","sighashtype"} }, + { "wallet", "unloadwallet", &unloadwallet, {"wallet_name"} }, { "wallet", "walletlock", &walletlock, {} }, { "wallet", "walletpassphrasechange", &walletpassphrasechange, {"oldpassphrase","newpassphrase"} }, { "wallet", "walletpassphrase", &walletpassphrase, {"passphrase","timeout"} }, diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 78cacc0206..6255f28b17 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -79,6 +79,15 @@ std::shared_ptr<CWallet> GetWallet(const std::string& name) return nullptr; } +// Custom deleter for shared_ptr<CWallet>. +static void ReleaseWallet(CWallet* wallet) +{ + LogPrintf("Releasing wallet %s\n", wallet->GetName()); + wallet->BlockUntilSyncedToCurrentChain(); + wallet->Flush(); + delete wallet; +} + const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; const uint256 CMerkleTx::ABANDON_HASH(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); @@ -923,11 +932,10 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) CWalletTx& wtx = (*ret.first).second; wtx.BindWallet(this); bool fInsertedNew = ret.second; - if (fInsertedNew) - { + if (fInsertedNew) { wtx.nTimeReceived = GetAdjustedTime(); wtx.nOrderPos = IncOrderPosNext(&batch); - wtxOrdered.insert(std::make_pair(wtx.nOrderPos, TxPair(&wtx, nullptr))); + wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, TxPair(&wtx, nullptr))); wtx.nTimeSmart = ComputeTimeSmart(wtx); AddToSpends(hash); } @@ -998,9 +1006,12 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) bool CWallet::LoadToWallet(const CWalletTx& wtxIn) { uint256 hash = wtxIn.GetHash(); - CWalletTx& wtx = mapWallet.emplace(hash, wtxIn).first->second; + const auto& ins = mapWallet.emplace(hash, wtxIn); + CWalletTx& wtx = ins.first->second; wtx.BindWallet(this); - wtxOrdered.insert(std::make_pair(wtx.nOrderPos, TxPair(&wtx, nullptr))); + if (/* insertion took place */ ins.second) { + wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, TxPair(&wtx, nullptr))); + } AddToSpends(hash); for (const CTxIn& txin : wtx.tx->vin) { auto it = mapWallet.find(txin.prevout.hash); @@ -1294,7 +1305,7 @@ void CWallet::BlockUntilSyncedToCurrentChain() { LOCK(cs_main); const CBlockIndex* initialChainTip = chainActive.Tip(); - if (m_last_block_processed->GetAncestor(initialChainTip->nHeight) == initialChainTip) { + if (m_last_block_processed && m_last_block_processed->GetAncestor(initialChainTip->nHeight) == initialChainTip) { return; } } @@ -3210,8 +3221,11 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 { AssertLockHeld(cs_wallet); // mapWallet DBErrors nZapSelectTxRet = WalletBatch(*database,"cr+").ZapSelectTx(vHashIn, vHashOut); - for (uint256 hash : vHashOut) - mapWallet.erase(hash); + for (uint256 hash : vHashOut) { + const auto& it = mapWallet.find(hash); + wtxOrdered.erase(it->second.m_it_wtxOrdered); + mapWallet.erase(it); + } if (nZapSelectTxRet == DBErrors::NEED_REWRITE) { @@ -3284,7 +3298,7 @@ bool CWallet::DelAddressBook(const CTxDestination& address) // Delete destdata tuples associated with address std::string strAddress = EncodeDestination(address); - for (const std::pair<std::string, std::string> &item : mapAddressBook[address].destdata) + for (const std::pair<const std::string, std::string> &item : mapAddressBook[address].destdata) { WalletBatch(*database).EraseDestData(strAddress, item.first); } @@ -3685,7 +3699,7 @@ std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) co { LOCK(cs_wallet); std::set<CTxDestination> result; - for (const std::pair<CTxDestination, CAddressBookData>& item : mapAddressBook) + for (const std::pair<const CTxDestination, CAddressBookData>& item : mapAddressBook) { const CTxDestination& address = item.first; const std::string& strName = item.second.name; @@ -4061,7 +4075,9 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(const std::string& name, int64_t nStart = GetTimeMillis(); bool fFirstRun = true; - std::shared_ptr<CWallet> walletInstance = std::make_shared<CWallet>(name, WalletDatabase::Create(path)); + // TODO: Can't use std::make_shared because we need a custom deleter but + // should be possible to use std::allocate_shared. + std::shared_ptr<CWallet> walletInstance(new CWallet(name, WalletDatabase::Create(path)), ReleaseWallet); DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun); if (nLoadWalletRet != DBErrors::LOAD_OK) { @@ -4090,8 +4106,6 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(const std::string& name, } } - uiInterface.LoadWallet(walletInstance); - int prev_version = walletInstance->nWalletVersion; if (gArgs.GetBoolArg("-upgradewallet", fFirstRun)) { @@ -4341,6 +4355,8 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(const std::string& name, } } + uiInterface.LoadWallet(walletInstance); + // Register with the validation interface. It's ok to do this after rescan since we're still holding cs_main. RegisterValidationInterface(walletInstance.get()); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 1ec2a9e771..b829394847 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -339,6 +339,7 @@ public: char fFromMe; std::string strFromAccount; int64_t nOrderPos; //!< position in ordered transaction list + std::multimap<int64_t, std::pair<CWalletTx*, CAccountingEntry*>>::const_iterator m_it_wtxOrdered; // memory only mutable bool fDebitCached; @@ -1099,6 +1100,9 @@ public: //! Flush wallet (bitdb flush) void Flush(bool shutdown=false); + /** Wallet is about to be unloaded */ + boost::signals2::signal<void ()> NotifyUnload; + /** * Address book entry changed. * @note called with lock cs_wallet held. diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 5a09142412..a925c70783 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -68,12 +68,12 @@ class AssumeValidTest(BitcoinTestFramework): def send_blocks_until_disconnected(self, p2p_conn): """Keep sending blocks to the node until we're disconnected.""" for i in range(len(self.blocks)): - if p2p_conn.state != "connected": + if not p2p_conn.is_connected: break try: p2p_conn.send_message(msg_block(self.blocks[i])) except IOError as e: - assert str(e) == 'Not connected, no pushbuf' + assert not p2p_conn.is_connected break def assert_blockchain_height(self, node, height): diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 8964c8d64b..6d51f31e35 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -36,7 +36,7 @@ class NotificationsTest(BitcoinTestFramework): wait_until(lambda: os.path.isfile(self.block_filename) and os.stat(self.block_filename).st_size >= (block_count * 65), timeout=10) # file content should equal the generated blocks hashes - with open(self.block_filename, 'r') as f: + with open(self.block_filename, 'r', encoding="utf-8") as f: assert_equal(sorted(blocks), sorted(l.strip() for l in f.read().splitlines())) self.log.info("test -walletnotify") @@ -45,7 +45,7 @@ class NotificationsTest(BitcoinTestFramework): # file content should equal the generated transaction hashes txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) - with open(self.tx_filename, 'r') as f: + with open(self.tx_filename, 'r', encoding="ascii") as f: assert_equal(sorted(txids_rpc), sorted(l.strip() for l in f.read().splitlines())) os.remove(self.tx_filename) @@ -58,7 +58,7 @@ class NotificationsTest(BitcoinTestFramework): # file content should equal the generated transaction hashes txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) - with open(self.tx_filename, 'r') as f: + with open(self.tx_filename, 'r', encoding="ascii") as f: assert_equal(sorted(txids_rpc), sorted(l.strip() for l in f.read().splitlines())) # Mine another 41 up-version blocks. -alertnotify should trigger on the 51st. diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 11a52b9ee2..d400507a66 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -260,10 +260,17 @@ class PruneTest(BitcoinTestFramework): # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000) assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) + # Save block transaction count before pruning, assert value + block1_details = node.getblock(node.getblockhash(1)) + assert_equal(block1_details["nTx"], len(block1_details["tx"])) + # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight) node.generate(6) assert_equal(node.getblockchaininfo()["blocks"], 1001) + # Pruned block should still know the number of transactions + assert_equal(node.getblockheader(node.getblockhash(1))["nTx"], block1_details["nTx"]) + # negative heights should raise an exception assert_raises_rpc_error(-8, "Negative", node.pruneblockchain, -10) diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index cb4c9867a3..37849e9213 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -87,7 +87,7 @@ class TestP2PConn(P2PInterface): This is used when we want to send a message into the node that we expect will get us disconnected, eg an invalid block.""" self.send_message(message) - wait_until(lambda: self.state != "connected", timeout=timeout, lock=mininode_lock) + wait_until(lambda: not self.is_connected, timeout=timeout, lock=mininode_lock) class CompactBlocksTest(BitcoinTestFramework): def set_test_params(self): diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 198dcc1490..dca5ea39de 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -118,11 +118,11 @@ class P2PLeakTest(BitcoinTestFramework): time.sleep(5) #This node should have been banned - assert no_version_bannode.state != "connected" + assert not no_version_bannode.is_connected # These nodes should have been disconnected - assert unsupported_service_bit5_node.state != "connected" - assert unsupported_service_bit7_node.state != "connected" + assert not unsupported_service_bit5_node.is_connected + assert not unsupported_service_bit7_node.is_connected self.nodes[0].disconnect_p2ps() diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 095cc4b734..7ee8168e2f 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -288,6 +288,7 @@ class SendHeadersTest(BitcoinTestFramework): # 1. Mine a block; expect inv announcements each time self.log.info("Part 1: headers don't start before sendheaders message...") for i in range(4): + self.log.debug("Part 1.{}: starting...".format(i)) old_tip = tip tip = self.mine_blocks(1) inv_node.check_last_inv_announcement(inv=[tip]) @@ -335,11 +336,13 @@ class SendHeadersTest(BitcoinTestFramework): height = self.nodes[0].getblockcount() + 1 block_time += 10 # Advance far enough ahead for i in range(10): + self.log.debug("Part 2.{}: starting...".format(i)) # Mine i blocks, and alternate announcing either via # inv (of tip) or via headers. After each, new blocks # mined by the node should successfully be announced # with block header, even though the blocks are never requested for j in range(2): + self.log.debug("Part 2.{}.{}: starting...".format(i, j)) blocks = [] for b in range(i + 1): blocks.append(create_block(tip, create_coinbase(height), block_time)) @@ -386,6 +389,7 @@ class SendHeadersTest(BitcoinTestFramework): # PART 3. Headers announcements can stop after large reorg, and resume after # getheaders or inv from peer. for j in range(2): + self.log.debug("Part 3.{}: starting...".format(j)) # First try mining a reorg that can propagate with header announcement new_block_hashes = self.mine_reorg(length=7) tip = new_block_hashes[-1] @@ -412,23 +416,28 @@ class SendHeadersTest(BitcoinTestFramework): test_node.wait_for_block(new_block_hashes[-1]) for i in range(3): + self.log.debug("Part 3.{}.{}: starting...".format(j, i)) + # Mine another block, still should get only an inv tip = self.mine_blocks(1) inv_node.check_last_inv_announcement(inv=[tip]) test_node.check_last_inv_announcement(inv=[tip]) if i == 0: - self.log.debug("Just get the data -- shouldn't cause headers announcements to resume") + # Just get the data -- shouldn't cause headers announcements to resume test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 1: - self.log.debug("Send a getheaders message that shouldn't trigger headers announcements to resume (best header sent will be too old)") + # Send a getheaders message that shouldn't trigger headers announcements + # to resume (best header sent will be too old) test_node.send_get_headers(locator=[fork_point], hashstop=new_block_hashes[1]) test_node.send_get_data([tip]) test_node.wait_for_block(tip) elif i == 2: + # This time, try sending either a getheaders to trigger resumption + # of headers announcements, or mine a new block and inv it, also + # triggering resumption of headers announcements. test_node.send_get_data([tip]) test_node.wait_for_block(tip) - self.log.debug("This time, try sending either a getheaders to trigger resumption of headers announcements, or mine a new block and inv it, also triggering resumption of headers announcements.") if j == 0: test_node.send_get_headers(locator=[tip], hashstop=0) test_node.sync_with_ping() @@ -532,6 +541,7 @@ class SendHeadersTest(BitcoinTestFramework): # First we test that receipt of an unconnecting header doesn't prevent # chain sync. for i in range(10): + self.log.debug("Part 5.{}: starting...".format(i)) test_node.last_message.pop("getdata", None) blocks = [] # Create two more blocks. diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index 6a21b693b4..e958536cf9 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -47,9 +47,9 @@ class TimeoutsTest(BitcoinTestFramework): sleep(1) - assert no_verack_node.connected - assert no_version_node.connected - assert no_send_node.connected + assert no_verack_node.is_connected + assert no_version_node.is_connected + assert no_send_node.is_connected no_verack_node.send_message(msg_ping()) no_version_node.send_message(msg_ping()) @@ -58,18 +58,18 @@ class TimeoutsTest(BitcoinTestFramework): assert "version" in no_verack_node.last_message - assert no_verack_node.connected - assert no_version_node.connected - assert no_send_node.connected + assert no_verack_node.is_connected + assert no_version_node.is_connected + assert no_send_node.is_connected no_verack_node.send_message(msg_ping()) no_version_node.send_message(msg_ping()) sleep(31) - assert not no_verack_node.connected - assert not no_version_node.connected - assert not no_send_node.connected + assert not no_verack_node.is_connected + assert not no_version_node.is_connected + assert not no_send_node.is_connected if __name__ == '__main__': TimeoutsTest().main() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 17e24453e5..7acc59c2c6 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -217,6 +217,7 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(header['confirmations'], 1) assert_equal(header['previousblockhash'], secondbesthash) assert_is_hex_string(header['chainwork']) + assert_equal(header['nTx'], 1) assert_is_hash_string(header['hash']) assert_is_hash_string(header['previousblockhash']) assert_is_hash_string(header['merkleroot']) diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index 060a2373b1..f573faaf17 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -81,11 +81,11 @@ class GetblockstatsTest(BitcoinTestFramework): 'mocktime': int(mocktime), 'stats': self.expected_stats, } - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: json.dump(to_dump, f, sort_keys=True, indent=2) def load_test_data(self, filename): - with open(filename, 'r') as f: + with open(filename, 'r', encoding="utf8") as f: d = json.load(f) blocks = d['blocks'] mocktime = d['mocktime'] diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 7c2125a177..5859f108a4 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -77,6 +77,12 @@ class P2PConnection(asyncore.dispatcher): super().__init__(map=mininode_socket_map) + self._conn_open = False + + @property + def is_connected(self): + return self._conn_open + def peer_connect(self, dstaddr, dstport, net="regtest"): self.dstaddr = dstaddr self.dstport = dstport @@ -84,7 +90,7 @@ class P2PConnection(asyncore.dispatcher): self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.sendbuf = b"" self.recvbuf = b"" - self.state = "connecting" + self._asyncore_pre_connection = True self.network = net self.disconnect = False @@ -97,22 +103,23 @@ class P2PConnection(asyncore.dispatcher): def peer_disconnect(self): # Connection could have already been closed by other end. - if self.state == "connected": - self.disconnect_node() + if self.is_connected: + self.disconnect = True # Signal asyncore to disconnect # Connection and disconnection methods def handle_connect(self): """asyncore callback when a connection is opened.""" - if self.state != "connected": + if not self.is_connected: logger.debug("Connected & Listening: %s:%d" % (self.dstaddr, self.dstport)) - self.state = "connected" + self._conn_open = True + self._asyncore_pre_connection = False self.on_open() def handle_close(self): """asyncore callback when a connection is closed.""" logger.debug("Closing connection to: %s:%d" % (self.dstaddr, self.dstport)) - self.state = "closed" + self._conn_open = False self.recvbuf = b"" self.sendbuf = b"" try: @@ -121,13 +128,6 @@ class P2PConnection(asyncore.dispatcher): pass self.on_close() - def disconnect_node(self): - """Disconnect the p2p connection. - - Called by the test logic thread. Causes the p2p connection - to be disconnected on the next iteration of the asyncore loop.""" - self.disconnect = True - # Socket read methods def handle_read(self): @@ -182,9 +182,8 @@ class P2PConnection(asyncore.dispatcher): def writable(self): """asyncore method to determine whether the handle_write() callback should be called on the next loop.""" with mininode_lock: - pre_connection = self.state == "connecting" length = len(self.sendbuf) - return (length > 0 or pre_connection) + return length > 0 or self._asyncore_pre_connection def handle_write(self): """asyncore callback when data should be written to the socket.""" @@ -192,7 +191,7 @@ class P2PConnection(asyncore.dispatcher): # asyncore does not expose socket connection, only the first read/write # event, thus we must check connection manually here to know when we # actually connect - if self.state == "connecting": + if self._asyncore_pre_connection: self.handle_connect() if not self.writable(): return @@ -204,26 +203,17 @@ class P2PConnection(asyncore.dispatcher): return self.sendbuf = self.sendbuf[sent:] - def send_message(self, message, pushbuf=False): + def send_message(self, message): """Send a P2P message over the socket. This method takes a P2P payload, builds the P2P header and adds the message to the send buffer to be sent over the socket.""" - if self.state != "connected" and not pushbuf: - raise IOError('Not connected, no pushbuf') + if not self.is_connected: + raise IOError('Not connected') self._log_message("send", message) - command = message.command - data = message.serialize() - tmsg = MAGIC_BYTES[self.network] - tmsg += command - tmsg += b"\x00" * (12 - len(command)) - tmsg += struct.pack("<I", len(data)) - th = sha256(data) - h = sha256(th) - tmsg += h[:4] - tmsg += data + tmsg = self._build_message(message) with mininode_lock: - if (len(self.sendbuf) == 0 and not pushbuf): + if len(self.sendbuf) == 0: try: sent = self.send(tmsg) self.sendbuf = tmsg[sent:] @@ -234,6 +224,20 @@ class P2PConnection(asyncore.dispatcher): # Class utility methods + def _build_message(self, message): + """Build a serialized P2P message""" + command = message.command + data = message.serialize() + tmsg = MAGIC_BYTES[self.network] + tmsg += command + tmsg += b"\x00" * (12 - len(command)) + tmsg += struct.pack("<I", len(data)) + th = sha256(data) + h = sha256(th) + tmsg += h[:4] + tmsg += data + return tmsg + def _log_message(self, direction, msg): """Logs a message being sent or received over the connection.""" if direction == "send": @@ -280,7 +284,7 @@ class P2PInterface(P2PConnection): vt.addrTo.port = self.dstport vt.addrFrom.ip = "0.0.0.0" vt.addrFrom.port = 0 - self.send_message(vt, True) + self.sendbuf = self._build_message(vt) # Will be sent right after handle_connect # Message receiving methods @@ -348,7 +352,7 @@ class P2PInterface(P2PConnection): # Connection helper methods def wait_for_disconnect(self, timeout=60): - test_function = lambda: self.state != "connected" + test_function = lambda: not self.is_connected wait_until(test_function, timeout=timeout, lock=mininode_lock) # Message receiving helper methods diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index e016148f70..5e0b61b5e7 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -327,7 +327,7 @@ def get_auth_cookie(datadir): assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): - with open(os.path.join(datadir, "regtest", ".cookie"), 'r') as f: + with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f: userpass = f.read() split_userpass = userpass.split(':') user = split_userpass[0] diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 5b3a4df0f9..36101d9f57 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -213,7 +213,7 @@ def main(): # Read config generated by configure. config = configparser.ConfigParser() configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" - config.read_file(open(configfile)) + config.read_file(open(configfile, encoding="utf8")) passon_args.append("--configfile=%s" % configfile) @@ -590,7 +590,7 @@ class RPCCoverage(): if not os.path.isfile(coverage_ref_filename): raise RuntimeError("No coverage reference found") - with open(coverage_ref_filename, 'r') as coverage_ref_file: + with open(coverage_ref_filename, 'r', encoding="utf8") as coverage_ref_file: all_cmds.update([line.strip() for line in coverage_ref_file.readlines()]) for root, dirs, files in os.walk(self.dir): @@ -599,7 +599,7 @@ class RPCCoverage(): coverage_filenames.add(os.path.join(root, filename)) for filename in coverage_filenames: - with open(filename, 'r') as coverage_file: + with open(filename, 'r', encoding="utf8") as coverage_file: covered_cmds.update([line.strip() for line in coverage_file.readlines()]) return all_cmds - covered_cmds diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 53638615f6..3cefd83459 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -88,7 +88,7 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist') # should not initialize if the specified walletdir is not a directory not_a_dir = wallet_dir('notadir') - open(not_a_dir, 'a').close() + open(not_a_dir, 'a', encoding="utf8").close() self.nodes[0].assert_start_raises_init_error(['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') self.log.info("Do not allow -zapwallettxes with multiwallet") @@ -234,5 +234,32 @@ class MultiWalletTest(BitcoinTestFramework): assert new_wallet_name in self.nodes[0].listwallets() + self.log.info("Test dynamic wallet unloading") + + # Test `unloadwallet` errors + assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].unloadwallet) + assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", self.nodes[0].unloadwallet, "dummy") + assert_raises_rpc_error(-18, "Requested wallet does not exist or is not loaded", node.get_wallet_rpc("dummy").unloadwallet) + assert_raises_rpc_error(-8, "Cannot unload the requested wallet", w1.unloadwallet, "w2"), + + # Successfully unload the specified wallet name + self.nodes[0].unloadwallet("w1") + assert 'w1' not in self.nodes[0].listwallets() + + # Successfully unload the wallet referenced by the request endpoint + w2.unloadwallet() + assert 'w2' not in self.nodes[0].listwallets() + + # Successfully unload all wallets + for wallet_name in self.nodes[0].listwallets(): + self.nodes[0].unloadwallet(wallet_name) + assert_equal(self.nodes[0].listwallets(), []) + assert_raises_rpc_error(-32601, "Method not found (wallet method is disabled because no wallet is loaded)", self.nodes[0].getwalletinfo) + + # Successfully load a previously unloaded wallet + self.nodes[0].loadwallet('w1') + assert_equal(self.nodes[0].listwallets(), ['w1']) + assert_equal(w1.getwalletinfo()['walletname'], 'w1') + if __name__ == '__main__': MultiWalletTest().main() diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py index 7e96852c5c..c3cdeef580 100755 --- a/test/lint/check-rpc-mappings.py +++ b/test/lint/check-rpc-mappings.py @@ -44,7 +44,7 @@ def process_commands(fname): """Find and parse dispatch table in implementation file `fname`.""" cmds = [] in_rpcs = False - with open(fname, "r") as f: + with open(fname, "r", encoding="utf8") as f: for line in f: line = line.rstrip() if not in_rpcs: @@ -70,7 +70,7 @@ def process_mapping(fname): """Find and parse conversion table in implementation file `fname`.""" cmds = [] in_rpcs = False - with open(fname, "r") as f: + with open(fname, "r", encoding="utf8") as f: for line in f: line = line.rstrip() if not in_rpcs: diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index 1c9dbc7f68..f1327469f3 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -11,6 +11,7 @@ # The resulting script should exactly transform the previous commit into the current # one. Any remaining diff signals an error. +export LC_ALL=C if test "x$1" = "x"; then echo "Usage: $0 <commit>..." exit 1 diff --git a/test/lint/git-subtree-check.sh b/test/lint/git-subtree-check.sh index 184951715e..85e8b841b6 100755 --- a/test/lint/git-subtree-check.sh +++ b/test/lint/git-subtree-check.sh @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C DIR="$1" COMMIT="$2" if [ -z "$COMMIT" ]; then diff --git a/test/lint/lint-all.sh b/test/lint/lint-all.sh index b6d86959c6..c9d4ec7199 100755 --- a/test/lint/lint-all.sh +++ b/test/lint/lint-all.sh @@ -7,6 +7,10 @@ # This script runs all contrib/devtools/lint-*.sh files, and fails if any exit # with a non-zero status code. +# This script is intentionally locale dependent by not setting "export LC_ALL=C" +# in order to allow for the executed lint scripts to opt in or opt out of locale +# dependence themselves. + set -u SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") diff --git a/test/lint/lint-filenames.sh b/test/lint/lint-filenames.sh new file mode 100755 index 0000000000..61e978fe79 --- /dev/null +++ b/test/lint/lint-filenames.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure only lowercase alphanumerics (a-z0-9), underscores (_), +# hyphens (-) and dots (.) are used in source code filenames. + +export LC_ALL=C + +EXIT_CODE=0 +OUTPUT=$(git ls-files -- "*.cpp" "*.h" "*.py" "*.sh" | grep -vE '^[a-z0-9_./-]+$' | grep -vE 'src/(secp256k1|univalue)/') +if [[ ${OUTPUT} != "" ]]; then + echo "Use only lowercase alphanumerics (a-z0-9), underscores (_), hyphens (-) and dots (.)" + echo "in source code filenames:" + echo + echo "${OUTPUT}" + EXIT_CODE=1 +fi +exit ${EXIT_CODE} diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh index 6a0dd556bb..ea791b0176 100755 --- a/test/lint/lint-include-guards.sh +++ b/test/lint/lint-include-guards.sh @@ -6,6 +6,7 @@ # # Check include guards. +export LC_ALL=C HEADER_ID_PREFIX="BITCOIN_" HEADER_ID_SUFFIX="_H" diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index f5daf4f9ac..03350576a7 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -8,6 +8,7 @@ # Guard against accidental introduction of new Boost dependencies. # Check includes: Check for duplicate includes. Enforce bracket syntax includes. +export LC_ALL=C IGNORE_REGEXP="/(leveldb|secp256k1|univalue)/" filter_suffix() { diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 3144f2c841..98814ce4b9 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -1,5 +1,6 @@ #!/bin/bash +export LC_ALL=C KNOWN_VIOLATIONS=( "src/base58.cpp:.*isspace" "src/bitcoin-tx.cpp.*stoul" diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh index 35be13ec19..89e7fb8764 100755 --- a/test/lint/lint-logs.sh +++ b/test/lint/lint-logs.sh @@ -12,7 +12,7 @@ # There are some instances of LogPrintf() in comments. Those can be # ignored - +export LC_ALL=C UNTERMINATED_LOGS=$(git grep --extended-regexp "LogPrintf?\(" -- "*.cpp" | \ grep -v '\\n"' | \ grep -v "/\* Continued \*/" | \ diff --git a/test/lint/lint-python-shebang.sh b/test/lint/lint-python-shebang.sh index f5c5971c03..031487a91a 100755 --- a/test/lint/lint-python-shebang.sh +++ b/test/lint/lint-python-shebang.sh @@ -1,5 +1,7 @@ #!/bin/bash # Shebang must use python3 (not python or python2) + +export LC_ALL=C EXIT_CODE=0 for PYTHON_FILE in $(git ls-files -- "*.py"); do if [[ $(head -c 2 "${PYTHON_FILE}") == "#!" && diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh new file mode 100755 index 0000000000..d0ff38ea0a --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +export LC_ALL=C +EXIT_CODE=0 +OUTPUT=$(git grep " open(" -- "*.py" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") +if [[ ${OUTPUT} != "" ]]; then + echo "Python's open(...) seems to be used to open text files without explicitly" + echo "specifying encoding=\"utf8\":" + echo + echo "${OUTPUT}" + EXIT_CODE=1 +fi +exit ${EXIT_CODE} diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 7d3555b6d4..d9d46d86d5 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -6,6 +6,8 @@ # # Check for specified flake8 warnings in python files. +export LC_ALL=C + # E101 indentation contains mixed spaces and tabs # E112 expected an indented block # E113 unexpected indentation diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh new file mode 100755 index 0000000000..d78bac2d47 --- /dev/null +++ b/test/lint/lint-shell-locale.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure all shell scripts: +# a.) explicitly opt out of locale dependence using "export LC_ALL=C", or +# b.) explicitly opt in to locale dependence using the annotation below. + +export LC_ALL=C + +EXIT_CODE=0 +for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|univalue)/"); do + if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then + continue + fi + FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*|)$' "${SHELL_SCRIPT}" | head -1) + if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" ]]; then + echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}" + EXIT_CODE=1 + fi +done +exit ${EXIT_CODE} diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 5f5fa9a925..bf56db90c6 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -6,6 +6,10 @@ # # Check for shellcheck warnings in shell scripts. +# This script is intentionally locale dependent by not setting "export LC_ALL=C" +# to allow running certain versions of shellcheck that core dump when LC_ALL=C +# is set. + # Disabled warnings: # SC2001: See if you can use ${variable//search/replace} instead. # SC2004: $/${} is unnecessary on arithmetic variables. diff --git a/test/lint/lint-tests.sh b/test/lint/lint-tests.sh index ffc0660551..9d83547df4 100755 --- a/test/lint/lint-tests.sh +++ b/test/lint/lint-tests.sh @@ -6,6 +6,7 @@ # # Check the test suite naming conventions +export LC_ALL=C EXIT_CODE=0 NAMING_INCONSISTENCIES=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh index c5d43043d5..128aa10da5 100755 --- a/test/lint/lint-whitespace.sh +++ b/test/lint/lint-whitespace.sh @@ -8,6 +8,7 @@ # We can't run this check unless we know the commit range for the PR. +export LC_ALL=C while getopts "?" opt; do case $opt in ?) diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py index 30bd13d0dc..16371a6234 100755 --- a/test/util/bitcoin-util-test.py +++ b/test/util/bitcoin-util-test.py @@ -28,7 +28,7 @@ import sys def main(): config = configparser.ConfigParser() config.optionxform = str - config.readfp(open(os.path.join(os.path.dirname(__file__), "../config.ini"))) + config.readfp(open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8")) env_conf = dict(config.items('environment')) parser = argparse.ArgumentParser(description=__doc__) @@ -49,7 +49,7 @@ def main(): def bctester(testDir, input_basename, buildenv): """ Loads and parses the input file, runs all tests and reports results""" input_filename = os.path.join(testDir, input_basename) - raw_data = open(input_filename).read() + raw_data = open(input_filename, encoding="utf8").read() input_data = json.loads(raw_data) failed_testcases = [] @@ -86,7 +86,7 @@ def bctest(testDir, testObj, buildenv): inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) - inputData = open(filename).read() + inputData = open(filename, encoding="utf8").read() stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) @@ -97,7 +97,7 @@ def bctest(testDir, testObj, buildenv): outputFn = testObj['output_cmp'] outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) try: - outputData = open(os.path.join(testDir, outputFn)).read() + outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read() except: logging.error("Output file " + outputFn + " can not be opened") raise diff --git a/test/util/rpcauth-test.py b/test/util/rpcauth-test.py index 2456feb102..46e9fbc739 100755 --- a/test/util/rpcauth-test.py +++ b/test/util/rpcauth-test.py @@ -18,7 +18,7 @@ class TestRPCAuth(unittest.TestCase): config_path = os.path.abspath( os.path.join(os.sep, os.path.abspath(os.path.dirname(__file__)), "../config.ini")) - with open(config_path) as config_file: + with open(config_path, encoding="utf8") as config_file: config.read_file(config_file) sys.path.insert(0, os.path.dirname(config['environment']['RPCAUTH'])) self.rpcauth = importlib.import_module('rpcauth') |