diff options
60 files changed, 709 insertions, 596 deletions
diff --git a/.gitignore b/.gitignore index 25710978c3..809e851ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -133,3 +133,7 @@ db4/ # clang-check *.plist + +osx_volname +dist/ +*.background.tiff diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 268259c82d..1f4a2c081e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,25 +79,26 @@ about Git. The title of the pull request should be prefixed by the component or area that the pull request affects. Valid areas as: - - *Consensus* for changes to consensus critical code - - *Doc* for changes to the documentation - - *Qt* for changes to bitcoin-qt - - *Log* Changes to log messages - - *Mining* for changes to the mining code - - *Net* or *P2P* for changes to the peer-to-peer network code - - *Refactor* for structural changes that do not change behavior - - *RPC/REST/ZMQ* for changes to the RPC, REST or ZMQ APIs - - *Scripts and tools* for changes to the scripts and tools - - *Test* for changes to the bitcoin unit tests or QA tests - - *Utils and libraries* for changes to the utils and libraries - - *Wallet* for changes to the wallet code + - `consensus` for changes to consensus critical code + - `doc` for changes to the documentation + - `qt` or `gui` for changes to bitcoin-qt + - `log` for changes to log messages + - `mining` for changes to the mining code + - `net` or `p2p` for changes to the peer-to-peer network code + - `refactor` for structural changes that do not change behavior + - `rpc`, `rest` or `zmq` for changes to the RPC, REST or ZMQ APIs + - `script` for changes to the scripts and tools + - `test` for changes to the bitcoin unit tests or QA tests + - `util` or `lib` for changes to the utils or libraries + - `wallet` for changes to the wallet code + - `build` for changes to the GNU Autotools, reproducible builds or CI code Examples: - Consensus: Add new opcode for BIP-XXXX OP_CHECKAWESOMESIG - Net: Automatically create hidden service, listen on Tor - Qt: Add feed bump button - Log: Fix typo in log message + consensus: Add new opcode for BIP-XXXX OP_CHECKAWESOMESIG + net: Automatically create hidden service, listen on Tor + qt: Add feed bump button + log: Fix typo in log message Note that translations should not be submitted as pull requests, please see [Translation Process](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md) diff --git a/Makefile.am b/Makefile.am index 712f2a454d..8b1e2a6b5b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -310,4 +310,5 @@ clean-docs: clean-local: clean-docs rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP) rm -rf test/functional/__pycache__ test/functional/test_framework/__pycache__ test/cache share/rpcauth/__pycache__ + rm -rf osx_volname dist/ dpi36.background.tiff dpi72.background.tiff diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 3d1024c7a5..04fa02484f 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -120,18 +120,6 @@ If there are 'unsupported' symbols, the return value will be 1 a list like this .../64/test_bitcoin: symbol std::out_of_range::~out_of_range() from unsupported version GLIBCXX_3.4.15 .../64/test_bitcoin: symbol _ZNSt8__detail15_List_nod from unsupported version GLIBCXX_3.4.15 -update-translations.py -====================== - -Run this script from the root of the repository to update all translations from transifex. -It will do the following automatically: - -- fetch all translations -- post-process them into valid and committable format -- add missing translations to the build system (TODO) - -See doc/translation-process.md for more information. - circular-dependencies.py ======================== diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py deleted file mode 100755 index 1b9d3a4c27..0000000000 --- a/contrib/devtools/update-translations.py +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014 Wladimir J. van der Laan -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Run this script from the root of the repository to update all translations from -transifex. -It will do the following automatically: - -- fetch all translations using the tx tool -- post-process them into valid and committable format - - remove invalid control characters - - remove location tags (makes diffs less noisy) - -TODO: -- auto-add new translations to the build system according to the translation process -''' -import subprocess -import re -import sys -import os -import io -import xml.etree.ElementTree as ET - -# Name of transifex tool -TX = 'tx' -# Name of source language file -SOURCE_LANG = 'bitcoin_en.ts' -# Directory with locale files -LOCALE_DIR = 'src/qt/locale' -# Minimum number of messages for translation to be considered at all -MIN_NUM_MESSAGES = 10 -# Regexp to check for Bitcoin addresses -ADDRESS_REGEXP = re.compile('([13]|bc1)[a-zA-Z0-9]{30,}') - -def check_at_repository_root(): - if not os.path.exists('.git'): - print('No .git directory found') - print('Execute this script at the root of the repository', file=sys.stderr) - sys.exit(1) - -def fetch_all_translations(): - if subprocess.call([TX, 'pull', '-f', '-a']): - print('Error while fetching translations', file=sys.stderr) - sys.exit(1) - -def find_format_specifiers(s): - '''Find all format specifiers in a string.''' - pos = 0 - specifiers = [] - while True: - percent = s.find('%', pos) - if percent < 0: - break - specifiers.append(s[percent+1]) - pos = percent+2 - return specifiers - -def split_format_specifiers(specifiers): - '''Split format specifiers between numeric (Qt) and others (strprintf)''' - numeric = [] - other = [] - for s in specifiers: - if s in {'1','2','3','4','5','6','7','8','9'}: - numeric.append(s) - else: - other.append(s) - - # If both numeric format specifiers and "others" are used, assume we're dealing - # with a Qt-formatted message. In the case of Qt formatting (see https://doc.qt.io/qt-5/qstring.html#arg) - # only numeric formats are replaced at all. This means "(percentage: %1%)" is valid, without needing - # any kind of escaping that would be necessary for strprintf. Without this, this function - # would wrongly detect '%)' as a printf format specifier. - if numeric: - other = [] - - # numeric (Qt) can be present in any order, others (strprintf) must be in specified order - return set(numeric),other - -def sanitize_string(s): - '''Sanitize string for printing''' - return s.replace('\n',' ') - -def check_format_specifiers(source, translation, errors, numerus): - source_f = split_format_specifiers(find_format_specifiers(source)) - # assert that no source messages contain both Qt and strprintf format specifiers - # if this fails, go change the source as this is hacky and confusing! - assert(not(source_f[0] and source_f[1])) - try: - translation_f = split_format_specifiers(find_format_specifiers(translation)) - except IndexError: - errors.append("Parse error in translation for '%s': '%s'" % (sanitize_string(source), sanitize_string(translation))) - return False - else: - if source_f != translation_f: - if numerus and source_f == (set(), ['n']) and translation_f == (set(), []) and translation.find('%') == -1: - # Allow numerus translations to omit %n specifier (usually when it only has one possible value) - return True - errors.append("Mismatch between '%s' and '%s'" % (sanitize_string(source), sanitize_string(translation))) - return False - return True - -def all_ts_files(suffix=''): - for filename in os.listdir(LOCALE_DIR): - # process only language files, and do not process source language - if not filename.endswith('.ts'+suffix) or filename == SOURCE_LANG+suffix: - continue - if suffix: # remove provided suffix - filename = filename[0:-len(suffix)] - filepath = os.path.join(LOCALE_DIR, filename) - yield(filename, filepath) - -FIX_RE = re.compile(b'[\x00-\x09\x0b\x0c\x0e-\x1f]') -def remove_invalid_characters(s): - '''Remove invalid characters from translation string''' - return FIX_RE.sub(b'', s) - -# Override cdata escape function to make our output match Qt's (optional, just for cleaner diffs for -# comparison, disable by default) -_orig_escape_cdata = None -def escape_cdata(text): - text = _orig_escape_cdata(text) - text = text.replace("'", ''') - text = text.replace('"', '"') - return text - -def contains_bitcoin_addr(text, errors): - if text is not None and ADDRESS_REGEXP.search(text) is not None: - errors.append('Translation "%s" contains a bitcoin address. This will be removed.' % (text)) - return True - return False - -def postprocess_translations(reduce_diff_hacks=False): - print('Checking and postprocessing...') - - if reduce_diff_hacks: - global _orig_escape_cdata - _orig_escape_cdata = ET._escape_cdata - ET._escape_cdata = escape_cdata - - for (filename,filepath) in all_ts_files(): - os.rename(filepath, filepath+'.orig') - - have_errors = False - for (filename,filepath) in all_ts_files('.orig'): - # pre-fixups to cope with transifex output - parser = ET.XMLParser(encoding='utf-8') # need to override encoding because 'utf8' is not understood only 'utf-8' - with open(filepath + '.orig', 'rb') as f: - data = f.read() - # remove control characters; this must be done over the entire file otherwise the XML parser will fail - data = remove_invalid_characters(data) - tree = ET.parse(io.BytesIO(data), parser=parser) - - # iterate over all messages in file - root = tree.getroot() - for context in root.findall('context'): - for message in context.findall('message'): - numerus = message.get('numerus') == 'yes' - source = message.find('source').text - translation_node = message.find('translation') - # pick all numerusforms - if numerus: - translations = [i.text for i in translation_node.findall('numerusform')] - else: - translations = [translation_node.text] - - for translation in translations: - if translation is None: - continue - errors = [] - valid = check_format_specifiers(source, translation, errors, numerus) and not contains_bitcoin_addr(translation, errors) - - for error in errors: - print('%s: %s' % (filename, error)) - - if not valid: # set type to unfinished and clear string if invalid - translation_node.clear() - translation_node.set('type', 'unfinished') - have_errors = True - - # Remove location tags - for location in message.findall('location'): - message.remove(location) - - # Remove entire message if it is an unfinished translation - if translation_node.get('type') == 'unfinished': - context.remove(message) - - # check if document is (virtually) empty, and remove it if so - num_messages = 0 - for context in root.findall('context'): - for message in context.findall('message'): - num_messages += 1 - if num_messages < MIN_NUM_MESSAGES: - print('Removing %s, as it contains only %i messages' % (filepath, num_messages)) - continue - - # write fixed-up tree - # if diff reduction requested, replace some XML to 'sanitize' to qt formatting - if reduce_diff_hacks: - out = io.BytesIO() - tree.write(out, encoding='utf-8') - out = out.getvalue() - out = out.replace(b' />', b'/>') - with open(filepath, 'wb') as f: - f.write(out) - else: - tree.write(filepath, encoding='utf-8') - return have_errors - -if __name__ == '__main__': - check_at_repository_root() - fetch_all_translations() - postprocess_translations() - diff --git a/depends/README.md b/depends/README.md index 6dbe365545..ca542be13f 100644 --- a/depends/README.md +++ b/depends/README.md @@ -12,14 +12,18 @@ For example: make HOST=x86_64-w64-mingw32 -j4 -A prefix will be generated that's suitable for plugging into Bitcoin's -configure. In the above example, a dir named x86_64-w64-mingw32 will be +**Bitcoin's configure script by default will ignore the depends output.** In +order for it to pick up libraries, tools, and settings from the depends build, +you must point it at the appropriate `--prefix` directory generated by the +build. In the above example, a prefix dir named x86_64-w64-mingw32 will be created. To use it for Bitcoin: ./configure --prefix=`pwd`/depends/x86_64-w64-mingw32 Common `host-platform-triplets` for cross compilation are: +- `i686-pc-linux-gnu` for Linux 32 bit +- `x86_64-pc-linux-gnu` for x86 Linux - `x86_64-w64-mingw32` for Win64 - `x86_64-apple-darwin14` for macOS - `arm-linux-gnueabihf` for Linux ARM 32 bit diff --git a/depends/packages.md b/depends/packages.md index 2c592885b6..36c9967a0a 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -181,3 +181,18 @@ For us, it's much easier to just link a static `libsecondary` into a shared static or dynamic `libseconday`, that's not our concern. With a static `libseconday`, when we need to link `libprimary` into our executable, there's no dependency chain to worry about as `libprimary` has all the symbols. + +## Build targets: + +To build an individual package (useful for debugging), following build targets are available. + + make ${package} + make ${package}_fetched + make ${package}_extracted + make ${package}_preprocessed + make ${package}_configured + make ${package}_built + make ${package}_staged + make ${package}_postprocessed + make ${package}_cached + make ${package}_cached_checksum diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index 293631739d..ed21fbba33 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -6,7 +6,7 @@ $(package)_sha256_hash=b449a3e10c47e1d1c7a6ec6e2016cca73d3bd68fbbd4f0ae5cc6b573f $(package)_dependencies=freetype expat define $(package)_set_vars - $(package)_config_opts=--disable-docs --disable-static + $(package)_config_opts=--disable-docs --disable-static --disable-libxml2 --disable-iconv endef define $(package)_config_cmds diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index 0e4a60ad25..058416f793 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -5,8 +5,10 @@ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=fdd477320aeb5cdd67272838722d6b7d544887dfe7de46e1e7cc0c27c2bea4f2 $(package)_dependencies=xproto +# When updating this package, check the default value of +# --disable-xthreads. It is currently enabled. define $(package)_set_vars - $(package)_config_opts=--disable-shared + $(package)_config_opts=--disable-shared --disable-lint-library --without-lint $(package)_config_opts_linux=--with-pic endef diff --git a/depends/packages/libxcb.mk b/depends/packages/libxcb.mk index 90bc15f9f4..49d3e3d15b 100644 --- a/depends/packages/libxcb.mk +++ b/depends/packages/libxcb.mk @@ -6,7 +6,7 @@ $(package)_sha256_hash=98d9ab05b636dd088603b64229dd1ab2d2cc02ab807892e107d674f9c $(package)_dependencies=xcb_proto libXau define $(package)_set_vars -$(package)_config_opts=--disable-static +$(package)_config_opts=--disable-static --disable-build-docs --without-doxygen --without-launchd # Because we pass -qt-xcb to Qt, it will compile in a set of xcb helper libraries and extensions, # so we skip building all of the extensions here. # More info is available from: https://doc.qt.io/qt-5.9/linux-requirements.html diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index 56681b52a1..3bc2cb768c 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -5,7 +5,8 @@ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=efe5188b1ddbcbf98763b819b146be6a90481aac30cfc8d858ab78a19cde1fa5 define $(package)_set_vars -$(package)_config_opts=--disable-shared -without-tools --disable-sdltest +$(package)_config_opts=--disable-shared --without-tools --without-tests --disable-sdltest +$(package)_config_opts += --disable-gprof --disable-gcov --disable-mudflap $(package)_config_opts_linux=--with-pic endef diff --git a/depends/packages/rapidcheck.mk b/depends/packages/rapidcheck.mk index fa4fb3c782..a16fee270e 100644 --- a/depends/packages/rapidcheck.mk +++ b/depends/packages/rapidcheck.mk @@ -1,11 +1,11 @@ package=rapidcheck -$(package)_version=3eb9b4ff69f4ff2d9932e8f852c2b2a61d7c20d3 +$(package)_version=d9482c683429fe79122e3dcab14c9655874aeb8e $(package)_download_path=https://github.com/emil-e/rapidcheck/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=5fbf82755c9a647127e62563be079448ff8b1db9ca80a52a673dd9a88fdb714b +$(package)_sha256_hash=b9ee8955b175fd3c0757ebd887bb075541761af08b0c28391b7c6c0685351f6b define $(package)_config_cmds - cmake -DCMAKE_INSTALL_PREFIX=$($(package)_staging_dir)$(host_prefix) -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DRC_INSTALL_ALL_EXTRAS=ON + cmake -DCMAKE_INSTALL_PREFIX=$($(package)_staging_dir)$(host_prefix) -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DRC_ENABLE_BOOST_TEST=ON -B . endef define $(package)_preprocess_cmds diff --git a/depends/packages/xproto.mk b/depends/packages/xproto.mk index 23ad5ffa10..2462f3c647 100644 --- a/depends/packages/xproto.mk +++ b/depends/packages/xproto.mk @@ -5,7 +5,7 @@ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f define $(package)_set_vars -$(package)_config_opts=--disable-shared +$(package)_config_opts=--without-fop --without-xmlto --without-xsltproc --disable-specs endef define $(package)_preprocess_cmds diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index e69a37e093..9ac037ebb5 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -6,7 +6,9 @@ $(package)_sha256_hash=bcbabe1e2c7d0eec4ed612e10b94b112dd5f06fcefa994a0c79a45d83 $(package)_patches=0001-fix-build-with-older-mingw64.patch 0002-disable-pthread_set_name_np.patch define $(package)_set_vars - $(package)_config_opts=--without-docs --disable-shared --without-libsodium --disable-curve --disable-curve-keygen --disable-perf --disable-Werror + $(package)_config_opts=--without-docs --disable-shared --disable-curve --disable-curve-keygen --disable-perf --disable-Werror --disable-drafts + $(package)_config_opts += --without-libsodium --without-libgssapi_krb5 --without-pgm --without-norm --without-vmci + $(package)_config_opts += --disable-libunwind --disable-radix-tree --without-gcov $(package)_config_opts_linux=--with-pic $(package)_cxxflags=-std=c++11 endef diff --git a/doc/release-notes-0.18.1-16257.md b/doc/release-notes-0.18.1-16257.md deleted file mode 100644 index 21867b7fb2..0000000000 --- a/doc/release-notes-0.18.1-16257.md +++ /dev/null @@ -1,6 +0,0 @@ -Wallet changes --------------- -When creating a transaction with a fee above `-maxtxfee` (default 0.1 BTC), -the RPC commands `walletcreatefundedpsbt` and `fundrawtransaction` will now fail -instead of rounding down the fee. Beware that the `feeRate` argument is specified -in BTC per kilobyte, not satoshi per byte. diff --git a/doc/release-notes-13756.md b/doc/release-notes-13756.md deleted file mode 100644 index a500aceb0f..0000000000 --- a/doc/release-notes-13756.md +++ /dev/null @@ -1,41 +0,0 @@ -Coin selection --------------- - -### Reuse Avoidance - -A new wallet flag `avoid_reuse` has been added (default off). When enabled, -a wallet will distinguish between used and unused addresses, and default to not -use the former in coin selection. - -Rescanning the blockchain is required, to correctly mark previously -used destinations. - -Together with "avoid partial spends" (present as of Bitcoin v0.17), this -addresses a serious privacy issue where a malicious user can track spends by -peppering a previously paid to address with near-dust outputs, which would then -be inadvertently included in future payments. - -New RPCs --------- - -- A new `setwalletflag` RPC sets/unsets flags for an existing wallet. - - -Updated RPCs ------------- - -Several RPCs have been updated to include an "avoid_reuse" flag, used to control -whether already used addresses should be left out or included in the operation. -These include: - -- createwallet -- getbalance -- getbalances -- sendtoaddress - -In addition, `sendtoaddress` has been changed to avoid partial spends when `avoid_reuse` -is enabled (if not already enabled via the `-avoidpartialspends` command line flag), -as it would otherwise risk using up the "wrong" UTXO for an address reuse case. - -The listunspent RPC has also been updated to now include a "reused" bool, for nodes -with "avoid_reuse" enabled. diff --git a/doc/release-notes-14054.md b/doc/release-notes-14054.md deleted file mode 100644 index d8cad369c5..0000000000 --- a/doc/release-notes-14054.md +++ /dev/null @@ -1,7 +0,0 @@ -P2P changes ------------ - -BIP 61 reject messages were deprecated in v0.18. They are now disabled by -default, but can be enabled by setting the `-enablebip61` command line option. -BIP 61 reject messages will be removed entirely in a future version of -Bitcoin Core. diff --git a/doc/release-notes-14802.md b/doc/release-notes-14802.md deleted file mode 100644 index 1fcc38866a..0000000000 --- a/doc/release-notes-14802.md +++ /dev/null @@ -1,3 +0,0 @@ -RPC changes ------------ -The `getblockstats` RPC is faster for fee calculation by using BlockUndo data. Also, `-txindex` is no longer required and `getblockstats` works for all non-pruned blocks. diff --git a/doc/release-notes-14954.md b/doc/release-notes-14954.md deleted file mode 100644 index 4bb0fcca74..0000000000 --- a/doc/release-notes-14954.md +++ /dev/null @@ -1,3 +0,0 @@ -Build system changes --------------------- -Python >=3.5 is now required by all aspects of the project. This includes the build systems, test framework and linters. The previously supported minimum (3.4), was EOL in March 2019. See #14954 for more details.
\ No newline at end of file diff --git a/doc/release-notes-15006.md b/doc/release-notes-15006.md deleted file mode 100644 index 76ed3247a6..0000000000 --- a/doc/release-notes-15006.md +++ /dev/null @@ -1,4 +0,0 @@ -Miscellaneous RPC changes ------------- - -- `createwallet` can now create encrypted wallets if a non-empty passphrase is specified. diff --git a/doc/release-notes-15427.md b/doc/release-notes-15427.md deleted file mode 100644 index 25edfd4402..0000000000 --- a/doc/release-notes-15427.md +++ /dev/null @@ -1,9 +0,0 @@ -Updated RPCs ------------- - -The `utxoupdatepsbt` RPC method has been updated to take a `descriptors` -argument. When provided, input and output scripts and keys will be filled in -when known, and P2SH-witness inputs will be filled in from the UTXO set when a -descriptor is provided that shows they're spending segwit outputs. - -See the RPC help text for full details. diff --git a/doc/release-notes-15566.md b/doc/release-notes-15566.md deleted file mode 100644 index 49964d7550..0000000000 --- a/doc/release-notes-15566.md +++ /dev/null @@ -1,3 +0,0 @@ -Miscellaneous CLI Changes -------------------------- -- The `testnet` field in `bitcoin-cli -getinfo` has been renamed to `chain` and now returns the current network name as defined in BIP70 (main, test, regtest).
\ No newline at end of file diff --git a/doc/release-notes-15620.md b/doc/release-notes-15620.md deleted file mode 100644 index bf89a70a4e..0000000000 --- a/doc/release-notes-15620.md +++ /dev/null @@ -1,13 +0,0 @@ -Updated RPCs ------------- - -* The -maxtxfee setting no longer has any effect on non-wallet RPCs. - - The `sendrawtransaction` and `testmempoolaccept` RPC methods previously - accepted an `allowhighfees` parameter to fail the mempool acceptance in case - the transaction's fee would exceed the value of the command line argument - `-maxtxfee`. To uncouple the RPCs from the global option, they now have a - hardcoded default for the maximum transaction fee, that can be changed for - both RPCs on a per-call basis with the `maxfeerate` parameter. The - `allowhighfees` boolean option has been removed and replaced by the - `maxfeerate` numeric option. diff --git a/doc/release-notes-15637.md b/doc/release-notes-15637.md deleted file mode 100644 index 048d5e7218..0000000000 --- a/doc/release-notes-15637.md +++ /dev/null @@ -1,3 +0,0 @@ -RPC changes ------------ -In getmempoolancestors, getmempooldescendants, getmempoolentry and getrawmempool RPCs, to be consistent with the returned value and other RPCs such as getrawtransaction, vsize has been added and size is now deprecated. size will only be returned if bitcoind is started with `-deprecatedrpc=size`.
\ No newline at end of file diff --git a/doc/release-notes-15730.md b/doc/release-notes-15730.md deleted file mode 100644 index 7a4a60b1ee..0000000000 --- a/doc/release-notes-15730.md +++ /dev/null @@ -1,5 +0,0 @@ -RPC changes ------------ -The RPC `getwalletinfo` response now includes the `scanning` key with an object -if there is a scanning in progress or `false` otherwise. Currently the object -has the scanning duration and progress. diff --git a/doc/release-notes-15849.md b/doc/release-notes-15849.md deleted file mode 100644 index a1df31f250..0000000000 --- a/doc/release-notes-15849.md +++ /dev/null @@ -1,6 +0,0 @@ -Thread names in logs --------------------- - -On platforms supporting `thread_local`, log lines can be prefixed with the name -of the thread that caused the log. To enable this behavior, use -`-logthreadnames=1`. diff --git a/doc/release-notes-15993.md b/doc/release-notes-15993.md deleted file mode 100644 index 493c7126ee..0000000000 --- a/doc/release-notes-15993.md +++ /dev/null @@ -1,3 +0,0 @@ -Build system changes --------------------- -The minimum supported miniUPnPc API version is set to 10. This keeps compatibility with Ubuntu 16.04 LTS and Debian 8 `libminiupnpc-dev` packages. Please note, on Debian this package is still vulnerable to [CVE-2017-8798](https://security-tracker.debian.org/tracker/CVE-2017-8798) (in jessie only) and [CVE-2017-1000494](https://security-tracker.debian.org/tracker/CVE-2017-1000494) (both in jessie and in stretch). diff --git a/doc/release-notes-16060.md b/doc/release-notes-16060.md deleted file mode 100644 index 7e150d10e7..0000000000 --- a/doc/release-notes-16060.md +++ /dev/null @@ -1,15 +0,0 @@ -Low-level RPC changes ----------------------- - -- Soft fork reporting in the `getblockchaininfo` return object has been - updated. For full details, see the RPC help text. In summary: - - The `bip9_softforks` sub-object is no longer returned - - The `softforks` sub-object now returns an object keyed by soft fork name, - rather than an array - - Each softfork object in the `softforks` object contains a `type` value which - is either `buried` (for soft fork deployments where the activation height is - hard-coded into the client implementation), or `bip9` (for soft fork deployments - where activation is controlled by BIP 9 signaling). - -- `getblocktemplate` no longer returns a `rules` array containing `CSV` - and `segwit` (the BIP 9 deployments that are currently in active state). diff --git a/doc/release-notes-16383.md b/doc/release-notes-16383.md deleted file mode 100644 index 8015715167..0000000000 --- a/doc/release-notes-16383.md +++ /dev/null @@ -1,8 +0,0 @@ -RPC changes ------------ - -RPCs which have an `include_watchonly` argument or `includeWatching` -option now default to `true` for watch-only wallets. Affected RPCs -are: `getbalance`, `listreceivedbyaddress`, `listreceivedbylabel`, -`listtransactions`, `listsinceblock`, `gettransaction`, -`walletcreatefundedpsbt`, and `fundrawtransaction`. diff --git a/doc/release-notes-16394.md b/doc/release-notes-16394.md deleted file mode 100644 index f09cba4b6d..0000000000 --- a/doc/release-notes-16394.md +++ /dev/null @@ -1,4 +0,0 @@ -RPC changes ------------ -`createwallet` now returns a warning if an empty string is used as an encryption password, and does not encrypt the wallet, instead of raising an error. -This makes it easier to disable encryption but also specify other options when using the `bitcoin-cli` tool. diff --git a/doc/release-notes.md b/doc/release-notes.md index e0673ae7ee..04aab56a72 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -64,14 +64,51 @@ platform. Notable changes =============== +New user documentation +---------------------- + +- [Reduce memory](https://github.com/bitcoin/bitcoin/blob/master/doc/reduce-memory.md) + suggests configuration tweaks for running Bitcoin Core on systems with + limited memory. (#16339) + New RPCs -------- - `getbalances` returns an object with all balances (`mine`, `untrusted_pending` and `immature`). Please refer to the RPC help of `getbalances` for details. The new RPC is intended to replace - `getunconfirmedbalance` and the balance fields in `getwalletinfo`, as well as - `getbalance`. The old calls may be removed in a future version. + `getbalance`, `getunconfirmedbalance`, and the balance fields in + `getwalletinfo`. These old calls and fields may be removed in a + future version. (#15930, #16239) + +- `setwalletflag` sets and unsets wallet flags that enable or disable + features specific to that existing wallet, such as the new + `avoid_reuse` feature documented elsewhere in these release notes. + (#13756) + +- `getblockfilter` gets the BIP158 filter for the specified block. This + RPC is only enabled if block filters have been created using the + `-blockfilterindex` configuration option. (#14121) + +New settings +------------ + +- `-blockfilterindex` enables the creation of BIP158 block filters for + the entire blockchain. Filters will be created in the background and + currently use about 4 GiB of space. Note: this version of Bitcoin + Core does not serve block filters over the P2P network, although the + local user may obtain block filters using the `getblockfilter` RPC. + (#14121) + +Updated settings +---------------- + +- `whitebind` and `whitelist` now accept a list of permissions to + provide peers connecting using the indicated interfaces or IP + addresses. If no permissions are specified with an address or CIDR + network, the implicit default permissions are the same as previous + releases. See the `bitcoind -help` output for these two options for + details about the available permissions. (#16248) Updated RPCs ------------ @@ -79,21 +116,128 @@ Updated RPCs Note: some low-level RPC changes mainly useful for testing are described in the Low-level Changes section below. -- The `sendmany` RPC had an argument `minconf` that was not well specified and - would lead to RPC errors even when the wallet's coin selection would succeed. - The `sendtoaddress` RPC never had this check, so to normalize the behavior, - `minconf` is now ignored in `sendmany`. If the coin selection does not - succeed due to missing coins, it will still throw an RPC error. Be reminded - that coin selection is influenced by the `-spendzeroconfchange`, - `-limitancestorcount`, `-limitdescendantcount` and `-walletrejectlongchains` - command line arguments. +- `sendmany` no longer has a `minconf` argument. This argument was not + well specified and would lead to RPC errors even when the wallet's + coin selection succeeded. Users who want to influence coin selection + can use the existing `-spendzeroconfchange`, `-limitancestorcount`, + `-limitdescendantcount` and `-walletrejectlongchains` configuration + arguments. (#15596) + +- `getbalance` and `sendtoaddress`, plus the new RPCs `getbalances` and + `createwallet`, now accept an "avoid_reuse" parameter that controls + whether already used addresses should be included in the operation. + Additionally, `sendtoaddress` will avoid partial spends when + `avoid_reuse` is enabled even if this feature is not already enabled + via the `-avoidpartialspends` command line flag because not doing so + would risk using up the "wrong" UTXO for an address reuse case. + (#13756) + +- `listunspent` now returns a "reused" bool for each output if the + wallet flag "avoid_reuse" is enabled. (#13756) + +- `getblockstats` now uses BlockUndo data instead of the transaction + index, making it much faster, no longer dependent on the `-txindex` + configuration option, and functional for all non-pruned blocks. + (#14802) + +- `utxoupdatepsbt` now accepts a `descriptors` parameter that will fill + out input and output scripts and keys when known. P2SH-witness inputs + will be filled in from the UTXO set when a descriptor is provided that + shows they're spending segwit outputs. See the RPC help text for full + details. (#15427) + +- `sendrawtransaction` and `testmempoolaccept` no longer accept a + `allowhighfees` parameter to fail mempool acceptance if the + transaction fee exceedes the value of the configuration option + `-maxtxfee`. Now there is a hardcoded default maximum feerate that + can be changed when calling either RPC using a `maxfeerate` parameter. + (#15620) + +- `getmempoolancestors`, `getmempooldescendants`, `getmempoolentry`, and + `getrawmempool` no longer return a `size` field unless the + configuration option `-deprecatedrpc=size` is used. Instead a new + `vsize` field is returned with the transaction's virtual size + (consistent with other RPCs such as `getrawtransaction`). (#15637) + +- `getwalletinfo` now includes a `scanning` field that is either `false` + (no scanning) or an object with information about the duration and + progress of the wallet's scanning historical blocks for transactions + affecting its balances. (#15730) + +- `createwallet` accepts a new `passphrase` parameter. If set, this + will create the new wallet encrypted with the given passphrase. If + unset (the default) or set to an empty string, no encryption will be + used. (#16394) + +- `getmempoolentry` now provides a `weight` field containing the + transaction weight as defined in BIP141. (#16647) + +- `getdescriptorinfo` now returns an additional `checksum` field + containing the checksum for the unmodified descriptor provided by the + user (that is, before the descriptor is normalized for the + `descriptor` field). (#15986) + +- `walletcreatefundedpsbt` now signals BIP125 Replace-by-Fee if the + `-walletrbf` configuration option is set to true. (#15911) + +GUI changes +----------- + +- Provides bech32 addresses by default. The user may change the address + during invoice generation using a GUI toggle, or the default address + type may be changed by the `-addresstype` configuration option. + (#15711, #16497) + +Deprecated or removed configuration options +------------------------------------------- + +- `-mempoolreplacement` is removed, although default node behavior + remains the same. This option previously allowed the user to prevent + the node from accepting or relaying BIP125 transaction replacements. + This is different from the remaining configuration option + `-walletrbf`. (#16171) Deprecated or removed RPCs -------------------------- -- The `totalFee` option of the `bumpfee` RPC has been deprecated and will be - removed in 0.20. To continue using this option start with - `-deprecatedrpc=totalFee`. See the `bumpfee` RPC help text for more details. +- `bumpfee` no longer accepts a `totalFee` option unless the + configuration parameter `deprecatedrpc=totalFee` is specified. This + parameter will be fully removed in a subsequent release. (#15996) + +- `generate` is now removed after being deprecated in Bitcoin Core 0.18. + Use the `generatetoaddress` RPC instead. (#15492) + +P2P changes +----------- + +- BIP 61 reject messages were deprecated in v0.18. They are now disabled + by default, but can be enabled by setting the `-enablebip61` command + line option. BIP 61 reject messages will be removed entirely in a + future version of Bitcoin Core. (#14054) + +- To eliminate well-known denial-of-service vectors in Bitcoin Core, + especially for nodes with spinning disks, the default value for the + `-peerbloomfilters` configuration option has been changed to false. + This prevents Bitcoin Core from sending the BIP111 NODE_BLOOM service + flag, accepting BIP37 bloom filters, or serving merkle blocks or + transactions matching a bloom filter. Users who still want to provide + bloom filter support may either set the configuration option to true + to re-enable both BIP111 and BIP37 support or enable just BIP37 + support for specific peers using the updated `-whitelist` and + `-whitebind` configuration options described elsewhere in these + release notes. For the near future, lightweight clients using public + BIP111/BIP37 nodes should still be able to connect to older versions + of Bitcoin Core and nodes that have manually enabled BIP37 support, + but developers of such software should consider migrating to either + using specific BIP37 nodes or an alternative transaction filtering + system. (#16152) + +Miscellaneous CLI Changes +------------------------- + +- The `testnet` field in `bitcoin-cli -getinfo` has been renamed to + `chain` and now returns the current network name as defined in BIP70 + (main, test, regtest). (#15566) Low-level changes ================= @@ -101,24 +245,42 @@ Low-level changes RPC --- +- `getblockchaininfo` no longer returns a `bip9_softforks` object. + Instead, information has been moved into the `softforks` object and + an additional `type` field describes how Bitcoin Core determines + whether that soft fork is active (e.g. BIP9 or BIP90). See the RPC + help for details. (#16060) + +- `getblocktemplate` no longer returns a `rules` array containing `CSV` + and `segwit` (the BIP9 deployments that are currently in active + state). (#16060) + +- `getrpcinfo` now returns a `logpath` field with the path to + `debug.log`. (#15483) Tests ----- -- The regression test chain, that can be enabled by the `-regtest` command line - flag, now requires transactions to not violate standard policy by default. - Making the default the same as for mainnet, makes it easier to test mainnet - behavior on regtest. Be reminded that the testnet still allows non-standard - txs by default and that the policy can be locally adjusted with the - `-acceptnonstdtxn` command line flag for both test chains. +- The regression test chain enabled by the `-regtest` command line flag + now requires transactions to not violate standard policy by default. + This is the same default used for mainnet and makes it easier to test + mainnet behavior on regtest. Note that the testnet still allows + non-standard txs by default and that the policy can be locally + adjusted with the `-acceptnonstdtxn` command line flag for both test + chains. (#15891) Configuration ------------ -- An error is issued where previously a warning was issued when a setting in - the config file was specified in the default section, but not overridden for - the selected network. This change takes only effect if the selected network - is not mainnet. +- A setting specified in the default section but not also specified in a + network-specific section (e.g. testnet) will now produce a error + preventing startup instead of just a warning unless the network is + mainnet. This prevents settings intended for mainnet from being + applied to testnet or regtest. (#15629) + +- On platforms supporting `thread_local`, log lines can be prefixed with + the name of the thread that caused the log. To enable this behavior, + use `-logthreadnames=1`. (#15849) Network ------- @@ -129,17 +291,81 @@ Network peers' announcements were received. In this release, the download logic has changed to randomize the fetch order across peers and to prefer sending download requests to outbound peers over inbound peers. This fixes an issue - where inbound peers can prevent a node from getting a transaction. + where inbound peers could prevent a node from getting a transaction. + (#14897, #15834) + +- If a Tor hidden service is being used, Bitcoin Core will be bound to + the standard port 8333 even if a different port is configured for + clearnet connections. This prevents leaking node identity through use + of identical non-default port numbers. (#15651) + +Mempool and transaction relay +----------------------------- + +- Allows one extra single-ancestor transaction per package. Previously, + if a transaction in the mempool had 25 descendants, or it and all of + its descendants were over 101,000 vbytes, any newly-received + transaction that was also a descendant would be ignored. Now, one + extra descendant will be allowed provided it is an immediate + descendant (child) and the child's size is 10,000 vbytes or less. + This makes it possible for two-party contract protocols such as + Lightning Network to give each participant an output they can spend + immediately for Child-Pays-For-Parent (CPFP) fee bumping without + allowing one malicious participant to fill the entire package and thus + prevent the other participant from spending their output. (#15681) + +- Transactions with outputs paying v1 to v16 witness versions (future + segwit versions) are now accepted into the mempool, relayed, and + mined. Attempting to spend those outputs remains forbidden by policy + ("non-standard"). When this change has been widely deployed, wallets + and services can accept any valid bech32 Bitcoin address without + concern that transactions paying future segwit versions will become + stuck in an unconfirmed state. (#15846) + +- Legacy transactions (transactions with no segwit inputs) must now be + sent using the legacy encoding format, enforcing the rule specified in + BIP144. (#14039) Wallet ------ - When in pruned mode, a rescan that was triggered by an `importwallet`, - `importpubkey`, `importaddress`, or `importprivkey` RPC will only fail when - blocks have been pruned. Previously it would fail when `-prune` has been set. - This change allows to set `-prune` to a high value (e.g. the disk size) and - the calls to any of the import RPCs would fail when the first block is - pruned. + `importpubkey`, `importaddress`, or `importprivkey` RPC will only fail + when blocks have been pruned. Previously it would fail when `-prune` + has been set. This change allows setting `-prune` to a high value + (e.g. the disk size) without the calls to any of the import RPCs + failing until the first block is pruned. (#15870) + +- When creating a transaction with a fee above `-maxtxfee` (default 0.1 + BTC), the RPC commands `walletcreatefundedpsbt` and + `fundrawtransaction` will now fail instead of rounding down the fee. + Be aware that the `feeRate` argument is specified in BTC per 1,000 + vbytes, not satoshi per vbyte. (#16257) + +- A new wallet flag `avoid_reuse` has been added (default off). When + enabled, a wallet will distinguish between used and unused addresses, + and default to not use the former in coin selection. When setting + this flag on an existing wallet, rescanning the blockchain is required + to correctly mark previously used destinations. Together with "avoid + partial spends" (added in Bitcoin Core v0.17.0), this can eliminate a + serious privacy issue where a malicious user can track spends by + sending small payments to a previously-paid address that would then + be included with unrelated inputs in future payments. (#13756) + +Build system changes +-------------------- + +- Python >=3.5 is now required by all aspects of the project. This + includes the build systems, test framework and linters. The previously + supported minimum (3.4), was EOL in March 2019. (#14954) + +- The minimum supported miniUPnPc API version is set to 10. This keeps + compatibility with Ubuntu 16.04 LTS and Debian 8 `libminiupnpc-dev` + packages. Please note, on Debian this package is still vulnerable to + [CVE-2017-8798](https://security-tracker.debian.org/tracker/CVE-2017-8798) + (in jessie only) and + [CVE-2017-1000494](https://security-tracker.debian.org/tracker/CVE-2017-1000494) + (both in jessie and in stretch). (#15993) Credits ======= diff --git a/doc/release-notes/release-notes-16152.md b/doc/release-notes/release-notes-16152.md deleted file mode 100644 index 9c77cb9ae5..0000000000 --- a/doc/release-notes/release-notes-16152.md +++ /dev/null @@ -1,7 +0,0 @@ -P2P Changes ------------ -- The default value for the -peerbloomfilters configuration option (and, thus, NODE_BLOOM support) has been changed to false. - This resolves well-known DoS vectors in Bitcoin Core, especially for nodes with spinning disks. It is not anticipated that - this will result in a significant lack of availability of NODE_BLOOM-enabled nodes in the coming years, however, clients - which rely on the availability of NODE_BLOOM-supporting nodes on the P2P network should consider the process of migrating - to a more modern (and less trustful and privacy-violating) alternative over the coming years. diff --git a/doc/translation_process.md b/doc/translation_process.md index b9a10b6527..0e9245250f 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -65,13 +65,13 @@ username = USERNAME The Transifex Bitcoin project config file is included as part of the repo. It can be found at `.tx/config`, however you shouldn’t need to change anything. ### Synchronising translations -To assist in updating translations, we have created a script to help. +To assist in updating translations, a helper script is available in the [maintainer-tools repo](https://github.com/bitcoin-core/bitcoin-maintainer-tools). -1. `python contrib/devtools/update-translations.py` +1. `python3 ../bitcoin-maintainer-tools/update-translations.py` 2. `git add` new translations from `src/qt/locale/` 3. Update `src/qt/bitcoin_locale.qrc` manually or via ```bash -git ls-files src/qt/locale/*ts|xargs -n1 basename|sed 's/\(bitcoin_\(.*\)\).ts/<file alias="\2">locale\/\1.qm<\/file>/' +git ls-files src/qt/locale/*ts|xargs -n1 basename|sed 's/\(bitcoin_\(.*\)\).ts/ <file alias="\2">locale\/\1.qm<\/file>/' ``` 4. Update `src/Makefile.qt.include` manually or via ```bash diff --git a/doc/zmq.md b/doc/zmq.md index 7ffc5623b6..a309abd0cc 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -111,7 +111,9 @@ using other means such as firewalling. Note that when the block chain tip changes, a reorganisation may occur and just the tip will be notified. It is up to the subscriber to -retrieve the chain from the last known block to the new tip. +retrieve the chain from the last known block to the new tip. Also note +that no notification occurs if the tip was in the active chain - this +is the case after calling invalidateblock RPC. There are several possibilities that ZMQ notification can get lost during transmission depending on the communication type you are diff --git a/src/Makefile.am b/src/Makefile.am index 141d8e68ea..8fc7f61d4b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -156,6 +156,7 @@ BITCOIN_CORE_H = \ netbase.h \ netmessagemaker.h \ node/coin.h \ + node/coinstats.h \ node/psbt.h \ node/transaction.h \ noui.h \ @@ -211,6 +212,7 @@ BITCOIN_CORE_H = \ util/memory.h \ util/moneystr.h \ util/rbf.h \ + util/string.h \ util/threadnames.h \ util/time.h \ util/translation.h \ @@ -277,6 +279,7 @@ libbitcoin_server_a_SOURCES = \ net.cpp \ net_processing.cpp \ node/coin.cpp \ + node/coinstats.cpp \ node/psbt.cpp \ node/transaction.cpp \ noui.cpp \ @@ -501,6 +504,7 @@ libbitcoin_util_a_SOURCES = \ util/rbf.cpp \ util/threadnames.cpp \ util/strencodings.cpp \ + util/string.cpp \ util/time.cpp \ util/url.cpp \ util/validation.cpp \ diff --git a/src/init.cpp b/src/init.cpp index dd43ef71c9..042335b8e7 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -438,9 +438,17 @@ void SetupServerArgs() #else hidden_args.emplace_back("-upnp"); #endif - gArgs.AddArg("-whitebind=<addr>", "Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - gArgs.AddArg("-whitelist=<IP address or network>", "Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or CIDR notated network (e.g. 1.2.3.0/24). Can be specified multiple times." - " Whitelisted peers cannot be DoS banned", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + gArgs.AddArg("-whitebind=<[permissions@]addr>", "Bind to given address and whitelist peers connecting to it. " + "Use [host]:port notation for IPv6. Allowed permissions are bloomfilter (allow requesting BIP37 filtered blocks and transactions), " + "noban (do not ban for misbehavior), " + "forcerelay (relay even non-standard transactions), " + "relay (relay even in -blocksonly mode), " + "and mempool (allow requesting BIP35 mempool contents). " + "Specify multiple permissions separated by commas (default: noban,mempool,relay). Can be specified multiple times.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + + gArgs.AddArg("-whitelist=<[permissions@]IP address or network>", "Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or " + "CIDR notated network(e.g. 1.2.3.0/24). Uses same permissions as " + "-whitebind. Can be specified multiple times." , ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); g_wallet_init_interface.AddWalletOptions(); @@ -510,8 +518,8 @@ void SetupServerArgs() gArgs.AddArg("-datacarriersize", strprintf("Maximum size of data in data carrier transactions we relay and mine (default: %u)", MAX_OP_RETURN_RELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); gArgs.AddArg("-minrelaytxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); - gArgs.AddArg("-whitelistforcerelay", strprintf("Force relay of transactions from whitelisted inbound peers even if the transactions were already in the mempool or violate local relay policy (default: %d)", DEFAULT_WHITELISTFORCERELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); - gArgs.AddArg("-whitelistrelay", strprintf("Accept relayed transactions received from whitelisted inbound peers even when not relaying transactions (default: %d)", DEFAULT_WHITELISTRELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); + gArgs.AddArg("-whitelistforcerelay", strprintf("Add 'forcerelay' permission to whitelisted inbound peers with default permissions. This will relay transactions even if the transactions were already in the mempool or violate local relay policy. (default: %d)", DEFAULT_WHITELISTFORCERELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); + gArgs.AddArg("-whitelistrelay", strprintf("Add 'relay' permission to whitelisted inbound peers with default permissions. The will accept relayed transactions even when not relaying transactions (default: %d)", DEFAULT_WHITELISTRELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); gArgs.AddArg("-blockmaxweight=<n>", strprintf("Set maximum BIP141 block weight (default: %d)", DEFAULT_BLOCK_MAX_WEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::BLOCK_CREATION); diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp new file mode 100644 index 0000000000..e1891b9898 --- /dev/null +++ b/src/node/coinstats.cpp @@ -0,0 +1,77 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/coinstats.h> + +#include <amount.h> +#include <coins.h> +#include <chain.h> +#include <hash.h> +#include <serialize.h> +#include <validation.h> +#include <uint256.h> +#include <util/system.h> + +#include <map> + +#include <boost/thread.hpp> + + +static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +{ + assert(!outputs.empty()); + ss << hash; + ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase ? 1u : 0u); + stats.nTransactions++; + for (const auto& output : outputs) { + ss << VARINT(output.first + 1); + ss << output.second.out.scriptPubKey; + ss << VARINT(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); + stats.nTransactionOutputs++; + stats.nTotalAmount += output.second.out.nValue; + stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + + 2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */; + } + ss << VARINT(0u); +} + +//! Calculate statistics about the unspent transaction output set +bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) +{ + std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); + assert(pcursor); + + CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + stats.hashBlock = pcursor->GetBestBlock(); + { + LOCK(cs_main); + stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; + } + ss << stats.hashBlock; + uint256 prevkey; + std::map<uint32_t, Coin> outputs; + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + COutPoint key; + Coin coin; + if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { + if (!outputs.empty() && key.hash != prevkey) { + ApplyStats(stats, ss, prevkey, outputs); + outputs.clear(); + } + prevkey = key.hash; + outputs[key.n] = std::move(coin); + } else { + return error("%s: unable to read value", __func__); + } + pcursor->Next(); + } + if (!outputs.empty()) { + ApplyStats(stats, ss, prevkey, outputs); + } + stats.hashSerialized = ss.GetHash(); + stats.nDiskSize = view->EstimateSize(); + return true; +} diff --git a/src/node/coinstats.h b/src/node/coinstats.h new file mode 100644 index 0000000000..7c11aab8bd --- /dev/null +++ b/src/node/coinstats.h @@ -0,0 +1,33 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_COINSTATS_H +#define BITCOIN_NODE_COINSTATS_H + +#include <amount.h> +#include <uint256.h> + +#include <cstdint> + +class CCoinsView; + +struct CCoinsStats +{ + int nHeight; + uint256 hashBlock; + uint64_t nTransactions; + uint64_t nTransactionOutputs; + uint64_t nBogoSize; + uint256 hashSerialized; + uint64_t nDiskSize; + CAmount nTotalAmount; + + CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} +}; + +//! Calculate statistics about the unspent transaction output set +bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats); + +#endif // BITCOIN_NODE_COINSTATS_H diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 29423db3d0..131cceccbe 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -10,6 +10,8 @@ #include <key_io.h> #include <wallet/wallet.h> +#include <algorithm> + #include <QFont> #include <QDebug> @@ -86,18 +88,18 @@ public: QString::fromStdString(EncodeDestination(address.dest)))); } } - // qLowerBound() and qUpperBound() require our cachedAddressTable list to be sorted in asc order + // std::lower_bound() and std::upper_bound() require our cachedAddressTable list to be sorted in asc order // Even though the map is already sorted this re-sorting step is needed because the originating map // is sorted by binary address, not by base58() address. - qSort(cachedAddressTable.begin(), cachedAddressTable.end(), AddressTableEntryLessThan()); + std::sort(cachedAddressTable.begin(), cachedAddressTable.end(), AddressTableEntryLessThan()); } void updateEntry(const QString &address, const QString &label, bool isMine, const QString &purpose, int status) { // Find address / label in model - QList<AddressTableEntry>::iterator lower = qLowerBound( + QList<AddressTableEntry>::iterator lower = std::lower_bound( cachedAddressTable.begin(), cachedAddressTable.end(), address, AddressTableEntryLessThan()); - QList<AddressTableEntry>::iterator upper = qUpperBound( + QList<AddressTableEntry>::iterator upper = std::upper_bound( cachedAddressTable.begin(), cachedAddressTable.end(), address, AddressTableEntryLessThan()); int lowerIndex = (lower - cachedAddressTable.begin()); int upperIndex = (upper - cachedAddressTable.begin()); diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index 8a6b205cd8..efc726e09e 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -10,6 +10,8 @@ #include <sync.h> #include <util/time.h> +#include <algorithm> + #include <QDebug> #include <QList> @@ -61,7 +63,7 @@ public: if (sortColumn >= 0) // sort cachedBanlist (use stable sort to prevent rows jumping around unnecessarily) - qStableSort(cachedBanlist.begin(), cachedBanlist.end(), BannedNodeLessThan(sortColumn, sortOrder)); + std::stable_sort(cachedBanlist.begin(), cachedBanlist.end(), BannedNodeLessThan(sortColumn, sortOrder)); } int size() const diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index 85b691c470..99a9a12fe2 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -11,6 +11,8 @@ #include <interfaces/node.h> #include <sync.h> +#include <algorithm> + #include <QDebug> #include <QList> #include <QTimer> @@ -76,7 +78,7 @@ public: if (sortColumn >= 0) // sort cacheNodeStats (use stable sort to prevent rows jumping around unnecessarily) - qStableSort(cachedNodeStats.begin(), cachedNodeStats.end(), NodeLessThan(sortColumn, sortOrder)); + std::stable_sort(cachedNodeStats.begin(), cachedNodeStats.end(), NodeLessThan(sortColumn, sortOrder)); // build index map mapNodeRows.clear(); diff --git a/src/qt/platformstyle.cpp b/src/qt/platformstyle.cpp index fca2a4e8c5..08d692e44c 100644 --- a/src/qt/platformstyle.cpp +++ b/src/qt/platformstyle.cpp @@ -114,11 +114,6 @@ QIcon PlatformStyle::SingleColorIcon(const QIcon& icon) const return ColorizeIcon(icon, SingleColor()); } -QIcon PlatformStyle::TextColorIcon(const QString& filename) const -{ - return ColorizeIcon(filename, TextColor()); -} - QIcon PlatformStyle::TextColorIcon(const QIcon& icon) const { return ColorizeIcon(icon, TextColor()); diff --git a/src/qt/platformstyle.h b/src/qt/platformstyle.h index 4e763e760e..635aec4c93 100644 --- a/src/qt/platformstyle.h +++ b/src/qt/platformstyle.h @@ -33,9 +33,6 @@ public: /** Colorize an icon (given object) with the icon color */ QIcon SingleColorIcon(const QIcon& icon) const; - /** Colorize an icon (given filename) with the text color */ - QIcon TextColorIcon(const QString& filename) const; - /** Colorize an icon (given object) with the text color */ QIcon TextColorIcon(const QIcon& icon) const; diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index aa746017f3..1611ec823c 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -11,6 +11,8 @@ #include <clientversion.h> #include <streams.h> +#include <algorithm> + RecentRequestsTableModel::RecentRequestsTableModel(WalletModel *parent) : QAbstractTableModel(parent), walletModel(parent) @@ -202,7 +204,7 @@ void RecentRequestsTableModel::addNewRequest(RecentRequestEntry &recipient) void RecentRequestsTableModel::sort(int column, Qt::SortOrder order) { - qSort(list.begin(), list.end(), RecentRequestEntryLessThan(column, order)); + std::sort(list.begin(), list.end(), RecentRequestEntryLessThan(column, order)); Q_EMIT dataChanged(index(0, 0, QModelIndex()), index(list.size() - 1, NUMBER_OF_COLUMNS - 1, QModelIndex())); } diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 193fba78b1..0ed057bc8b 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -230,8 +230,9 @@ void SendCoinsDialog::on_sendButton_clicked() { recipients.append(entry->getValue()); } - else + else if (valid) { + ui->scrollArea->ensureWidgetVisible(entry); valid = false; } } diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 1064c60dfd..8d0cb54151 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -17,6 +17,8 @@ #include <interfaces/handler.h> #include <uint256.h> +#include <algorithm> + #include <QColor> #include <QDateTime> #include <QDebug> @@ -93,9 +95,9 @@ public: qDebug() << "TransactionTablePriv::updateWallet: " + QString::fromStdString(hash.ToString()) + " " + QString::number(status); // Find bounds of this transaction in model - QList<TransactionRecord>::iterator lower = qLowerBound( + QList<TransactionRecord>::iterator lower = std::lower_bound( cachedWallet.begin(), cachedWallet.end(), hash, TxLessThan()); - QList<TransactionRecord>::iterator upper = qUpperBound( + QList<TransactionRecord>::iterator upper = std::upper_bound( cachedWallet.begin(), cachedWallet.end(), hash, TxLessThan()); int lowerIndex = (lower - cachedWallet.begin()); int upperIndex = (upper - cachedWallet.begin()); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index bfb6ab6a21..7161c39ffd 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -10,6 +10,7 @@ #include <chain.h> #include <chainparams.h> #include <coins.h> +#include <node/coinstats.h> #include <consensus/validation.h> #include <core_io.h> #include <hash.h> @@ -374,6 +375,7 @@ static std::string EntryDescriptionString() return " \"vsize\" : n, (numeric) virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted.\n" " \"size\" : n, (numeric) (DEPRECATED) same as vsize. Only returned if bitcoind is started with -deprecatedrpc=size\n" " size will be completely removed in v0.20.\n" + " \"weight\" : n, (numeric) transaction weight as defined in BIP 141.\n" " \"fee\" : n, (numeric) transaction fee in " + CURRENCY_UNIT + " (DEPRECATED)\n" " \"modifiedfee\" : n, (numeric) transaction fee with fee deltas used for mining priority (DEPRECATED)\n" " \"time\" : n, (numeric) local time transaction entered pool in seconds since 1 Jan 1970 GMT\n" @@ -413,6 +415,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("vsize", (int)e.GetTxSize()); if (IsDeprecatedRPCEnabled("size")) info.pushKV("size", (int)e.GetTxSize()); + info.pushKV("weight", (int)e.GetTxWeight()); info.pushKV("fee", ValueFromAmount(e.GetFee())); info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); info.pushKV("time", e.GetTime()); @@ -907,77 +910,6 @@ static UniValue getblock(const JSONRPCRequest& request) return blockToJSON(block, tip, pblockindex, verbosity >= 2); } -struct CCoinsStats -{ - int nHeight; - uint256 hashBlock; - uint64_t nTransactions; - uint64_t nTransactionOutputs; - uint64_t nBogoSize; - uint256 hashSerialized; - uint64_t nDiskSize; - CAmount nTotalAmount; - - CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {} -}; - -static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, const std::map<uint32_t, Coin>& outputs) -{ - assert(!outputs.empty()); - ss << hash; - ss << VARINT(outputs.begin()->second.nHeight * 2 + outputs.begin()->second.fCoinBase ? 1u : 0u); - stats.nTransactions++; - for (const auto& output : outputs) { - ss << VARINT(output.first + 1); - ss << output.second.out.scriptPubKey; - ss << VARINT(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); - stats.nTransactionOutputs++; - stats.nTotalAmount += output.second.out.nValue; - stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + - 2 /* scriptPubKey len */ + output.second.out.scriptPubKey.size() /* scriptPubKey */; - } - ss << VARINT(0u); -} - -//! Calculate statistics about the unspent transaction output set -static bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats) -{ - std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); - assert(pcursor); - - CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); - stats.hashBlock = pcursor->GetBestBlock(); - { - LOCK(cs_main); - stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; - } - ss << stats.hashBlock; - uint256 prevkey; - std::map<uint32_t, Coin> outputs; - while (pcursor->Valid()) { - boost::this_thread::interruption_point(); - COutPoint key; - Coin coin; - if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { - if (!outputs.empty() && key.hash != prevkey) { - ApplyStats(stats, ss, prevkey, outputs); - outputs.clear(); - } - prevkey = key.hash; - outputs[key.n] = std::move(coin); - } else { - return error("%s: unable to read value", __func__); - } - pcursor->Next(); - } - if (!outputs.empty()) { - ApplyStats(stats, ss, prevkey, outputs); - } - stats.hashSerialized = ss.GetHash(); - stats.nDiskSize = view->EstimateSize(); - return true; -} - static UniValue pruneblockchain(const JSONRPCRequest& request) { RPCHelpMan{"pruneblockchain", "", diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 464537a5b0..22d67c34da 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -4,11 +4,12 @@ #include <key_io.h> #include <outputtype.h> -#include <script/signingprovider.h> #include <rpc/util.h> #include <script/descriptor.h> +#include <script/signingprovider.h> #include <tinyformat.h> #include <util/strencodings.h> +#include <util/string.h> #include <tuple> @@ -645,11 +646,7 @@ std::string RPCArg::ToString(const bool oneline) const } case Type::OBJ: case Type::OBJ_USER_KEYS: { - std::string res; - for (size_t i = 0; i < m_inner.size();) { - res += m_inner[i].ToStringObj(oneline); - if (++i < m_inner.size()) res += ","; - } + const std::string res = Join(m_inner, ",", [&](const RPCArg& i) { return i.ToStringObj(oneline); }); if (m_type == Type::OBJ) { return "{" + res + "}"; } else { diff --git a/src/serialize.h b/src/serialize.h index 1dc27d84eb..a38d76fc18 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -555,6 +555,7 @@ template<typename Stream, unsigned int N, typename T> inline void Unserialize(St * vectors of unsigned char are a special case and are intended to be serialized as a single opaque blob. */ template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&); +template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&); template<typename Stream, typename T, typename A, typename V> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&); template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v); template<typename Stream, typename T, typename A> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&); @@ -713,6 +714,18 @@ void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char& os.write((char*)v.data(), v.size() * sizeof(T)); } +template<typename Stream, typename T, typename A> +void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&) +{ + // A special case for std::vector<bool>, as dereferencing + // std::vector<bool>::const_iterator does not result in a const bool& + // due to std::vector's special casing for bool arguments. + WriteCompactSize(os, v.size()); + for (bool elem : v) { + ::Serialize(os, elem); + } +} + template<typename Stream, typename T, typename A, typename V> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&) { diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 1b9ec76385..50ac0bd7b8 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -42,33 +42,47 @@ bool EqualDescriptor(std::string a, std::string b) return a == b; } -std::string MaybeUseHInsteadOfApostrophy(std::string ret) +std::string UseHInsteadOfApostrophe(const std::string& desc) { - if (InsecureRandBool()) { - while (true) { - auto it = ret.find("'"); - if (it != std::string::npos) { - ret[it] = 'h'; - if (ret.size() > 9 && ret[ret.size() - 9] == '#') ret = ret.substr(0, ret.size() - 9); // Changing apostrophe to h breaks the checksum - } else { - break; - } - } + std::string ret = desc; + while (true) { + auto it = ret.find('\''); + if (it == std::string::npos) break; + ret[it] = 'h'; + } + + // GetDescriptorChecksum returns "" if the checksum exists but is bad. + // Switching apostrophes with 'h' breaks the checksum if it exists - recalculate it and replace the broken one. + if (GetDescriptorChecksum(ret) == "") { + ret = ret.substr(0, desc.size() - 9); + ret += std::string("#") + GetDescriptorChecksum(ret); } return ret; } const std::set<std::vector<uint32_t>> ONLY_EMPTY{{}}; -void Check(const std::string& prv, const std::string& pub, int flags, const std::vector<std::vector<std::string>>& scripts, const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY) +void DoCheck(const std::string& prv, const std::string& pub, int flags, const std::vector<std::vector<std::string>>& scripts, const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY, + bool replace_apostrophe_with_h_in_prv=false, bool replace_apostrophe_with_h_in_pub=false) { FlatSigningProvider keys_priv, keys_pub; std::set<std::vector<uint32_t>> left_paths = paths; std::string error; + std::unique_ptr<Descriptor> parse_priv; + std::unique_ptr<Descriptor> parse_pub; // Check that parsing succeeds. - auto parse_priv = Parse(MaybeUseHInsteadOfApostrophy(prv), keys_priv, error); - auto parse_pub = Parse(MaybeUseHInsteadOfApostrophy(pub), keys_pub, error); + if (replace_apostrophe_with_h_in_prv) { + parse_priv = Parse(UseHInsteadOfApostrophe(prv), keys_priv, error); + } else { + parse_priv = Parse(prv, keys_priv, error); + } + if (replace_apostrophe_with_h_in_pub) { + parse_pub = Parse(UseHInsteadOfApostrophe(pub), keys_pub, error); + } else { + parse_pub = Parse(pub, keys_pub, error); + } + BOOST_CHECK(parse_priv); BOOST_CHECK(parse_pub); @@ -167,6 +181,32 @@ void Check(const std::string& prv, const std::string& pub, int flags, const std: BOOST_CHECK_MESSAGE(left_paths.empty(), "Not all expected key paths found: " + prv); } +void Check(const std::string& prv, const std::string& pub, int flags, const std::vector<std::vector<std::string>>& scripts, const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY) +{ + bool found_apostrophes_in_prv = false; + bool found_apostrophes_in_pub = false; + + // Do not replace apostrophes with 'h' in prv and pub + DoCheck(prv, pub, flags, scripts, paths); + + // Replace apostrophes with 'h' in prv but not in pub, if apostrophes are found in prv + if (prv.find('\'') != std::string::npos) { + found_apostrophes_in_prv = true; + DoCheck(prv, pub, flags, scripts, paths, /* replace_apostrophe_with_h_in_prv = */true, /*replace_apostrophe_with_h_in_pub = */false); + } + + // Replace apostrophes with 'h' in pub but not in prv, if apostrophes are found in pub + if (pub.find('\'') != std::string::npos) { + found_apostrophes_in_pub = true; + DoCheck(prv, pub, flags, scripts, paths, /* replace_apostrophe_with_h_in_prv = */false, /*replace_apostrophe_with_h_in_pub = */true); + } + + // Replace apostrophes with 'h' both in prv and in pub, if apostrophes are found in both + if (found_apostrophes_in_prv && found_apostrophes_in_pub) { + DoCheck(prv, pub, flags, scripts, paths, /* replace_apostrophe_with_h_in_prv = */true, /*replace_apostrophe_with_h_in_pub = */true); + } +} + } BOOST_FIXTURE_TEST_SUITE(descriptor_tests, BasicTestingSetup) diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 8a8620938e..b90be15fba 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -258,6 +258,14 @@ static bool isCanonicalException(const std::ios_base::failure& ex) return strcmp(expectedException.what(), ex.what()) == 0; } +BOOST_AUTO_TEST_CASE(vector_bool) +{ + std::vector<uint8_t> vec1{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; + std::vector<bool> vec2{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; + + BOOST_CHECK(vec1 == std::vector<uint8_t>(vec2.begin(), vec2.end())); + BOOST_CHECK(SerializeHash(vec1) == SerializeHash(vec2)); +} BOOST_AUTO_TEST_CASE(noncanonical) { diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 7119f56fc3..65cb956fbe 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -6,11 +6,12 @@ #include <clientversion.h> #include <sync.h> +#include <test/setup_common.h> #include <test/util.h> -#include <util/strencodings.h> #include <util/moneystr.h> +#include <util/strencodings.h> +#include <util/string.h> #include <util/time.h> -#include <test/setup_common.h> #include <stdint.h> #include <thread> @@ -123,6 +124,19 @@ BOOST_AUTO_TEST_CASE(util_HexStr) ); } +BOOST_AUTO_TEST_CASE(util_Join) +{ + // Normal version + BOOST_CHECK_EQUAL(Join({}, ", "), ""); + BOOST_CHECK_EQUAL(Join({"foo"}, ", "), "foo"); + BOOST_CHECK_EQUAL(Join({"foo", "bar"}, ", "), "foo, bar"); + + // Version with unary operator + const auto op_upper = [](const std::string& s) { return ToUpper(s); }; + BOOST_CHECK_EQUAL(Join<std::string>({}, ", ", op_upper), ""); + BOOST_CHECK_EQUAL(Join<std::string>({"foo"}, ", ", op_upper), "FOO"); + BOOST_CHECK_EQUAL(Join<std::string>({"foo", "bar"}, ", ", op_upper), "FOO, BAR"); +} BOOST_AUTO_TEST_CASE(util_FormatISO8601DateTime) { diff --git a/src/util/string.cpp b/src/util/string.cpp new file mode 100644 index 0000000000..8ea3a1afc6 --- /dev/null +++ b/src/util/string.cpp @@ -0,0 +1,5 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/string.h> diff --git a/src/util/string.h b/src/util/string.h new file mode 100644 index 0000000000..dec0c19b08 --- /dev/null +++ b/src/util/string.h @@ -0,0 +1,35 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_STRING_H +#define BITCOIN_UTIL_STRING_H + +#include <functional> +#include <string> +#include <vector> + +/** + * Join a list of items + * + * @param list The list to join + * @param separator The separator + * @param unary_op Apply this operator to each item in the list + */ +template <typename T, typename UnaryOp> +std::string Join(const std::vector<T>& list, const std::string& separator, UnaryOp unary_op) +{ + std::string ret; + for (size_t i = 0; i < list.size(); ++i) { + if (i > 0) ret += separator; + ret += unary_op(list.at(i)); + } + return ret; +} + +inline std::string Join(const std::vector<std::string>& list, const std::string& separator) +{ + return Join(list, separator, [](const std::string& i) { return i; }); +} + +#endif // BITCOIN_UTIL_STRENCODINGS_H diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index cf388ad827..984be3e301 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -444,7 +444,7 @@ public: * on this bitcoin node, and set to 0 for transactions that were created * externally and came in through the network or sendrawtransaction RPC. */ - char fFromMe; + bool fFromMe; int64_t nOrderPos; //!< position in ordered transaction list std::multimap<int64_t, CWalletTx*>::const_iterator m_it_wtxOrdered; @@ -501,8 +501,8 @@ public: std::vector<char> dummy_vector1; //!< Used to be vMerkleBranch std::vector<char> dummy_vector2; //!< Used to be vtxPrev - char dummy_char = false; //!< Used to be fSpent - s << tx << hashBlock << dummy_vector1 << nIndex << dummy_vector2 << mapValueCopy << vOrderForm << fTimeReceivedIsTxTime << nTimeReceived << fFromMe << dummy_char; + bool dummy_bool = false; //!< Used to be fSpent + s << tx << hashBlock << dummy_vector1 << nIndex << dummy_vector2 << mapValueCopy << vOrderForm << fTimeReceivedIsTxTime << nTimeReceived << fFromMe << dummy_bool; } template<typename Stream> @@ -512,8 +512,8 @@ public: std::vector<uint256> dummy_vector1; //!< Used to be vMerkleBranch std::vector<CMerkleTx> dummy_vector2; //!< Used to be vtxPrev - char dummy_char; //! Used to be fSpent - s >> tx >> hashBlock >> dummy_vector1 >> nIndex >> dummy_vector2 >> mapValue >> vOrderForm >> fTimeReceivedIsTxTime >> nTimeReceived >> fFromMe >> dummy_char; + bool dummy_bool; //! Used to be fSpent + s >> tx >> hashBlock >> dummy_vector1 >> nIndex >> dummy_vector2 >> mapValue >> vOrderForm >> fTimeReceivedIsTxTime >> nTimeReceived >> fFromMe >> dummy_bool; ReadOrderPos(nOrderPos, mapValue); nTimeSmart = mapValue.count("timesmart") ? (unsigned int)atoi64(mapValue["timesmart"]) : 0; diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index d47065d1cb..b9db618575 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -226,6 +226,16 @@ class SegWitTest(BitcoinTestFramework): assert tx.wit.is_null() # This should not be a segwit input assert txid1 in self.nodes[0].getrawmempool() + tx1_hex = self.nodes[0].gettransaction(txid1)['hex'] + tx1 = FromHex(CTransaction(), tx1_hex) + + # Check that wtxid is properly reported in mempool entry (txid1) + assert_equal(int(self.nodes[0].getmempoolentry(txid1)["wtxid"], 16), tx1.calc_sha256(True)) + + # Check that weight and vsize are properly reported in mempool entry (txid1) + assert_equal(self.nodes[0].getmempoolentry(txid1)["vsize"], (self.nodes[0].getmempoolentry(txid1)["weight"] + 3) // 4) + assert_equal(self.nodes[0].getmempoolentry(txid1)["weight"], len(tx1.serialize_without_witness())*3 + len(tx1.serialize_with_witness())) + # Now create tx2, which will spend from txid1. tx = CTransaction() tx.vin.append(CTxIn(COutPoint(int(txid1, 16), 0), b'')) @@ -235,6 +245,13 @@ class SegWitTest(BitcoinTestFramework): tx = FromHex(CTransaction(), tx2_hex) assert not tx.wit.is_null() + # Check that wtxid is properly reported in mempool entry (txid2) + assert_equal(int(self.nodes[0].getmempoolentry(txid2)["wtxid"], 16), tx.calc_sha256(True)) + + # Check that weight and vsize are properly reported in mempool entry (txid2) + assert_equal(self.nodes[0].getmempoolentry(txid2)["vsize"], (self.nodes[0].getmempoolentry(txid2)["weight"] + 3) // 4) + assert_equal(self.nodes[0].getmempoolentry(txid2)["weight"], len(tx.serialize_without_witness())*3 + len(tx.serialize_with_witness())) + # Now create tx3, which will spend from txid2 tx = CTransaction() tx.vin.append(CTxIn(COutPoint(int(txid2, 16), 0), b"")) @@ -251,9 +268,13 @@ class SegWitTest(BitcoinTestFramework): assert txid2 in template_txids assert txid3 in template_txids - # Check that wtxid is properly reported in mempool entry + # Check that wtxid is properly reported in mempool entry (txid3) assert_equal(int(self.nodes[0].getmempoolentry(txid3)["wtxid"], 16), tx.calc_sha256(True)) + # Check that weight and vsize are properly reported in mempool entry (txid3) + assert_equal(self.nodes[0].getmempoolentry(txid3)["vsize"], (self.nodes[0].getmempoolentry(txid3)["weight"] + 3) // 4) + assert_equal(self.nodes[0].getmempoolentry(txid3)["weight"], len(tx.serialize_without_witness())*3 + len(tx.serialize_with_witness())) + # Mine a block to clear the gbt cache again. self.nodes[0].generate(1) diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 1ba781c539..5aea10fbce 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -8,10 +8,9 @@ import struct from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import CTransaction, hash256 -from test_framework.util import assert_equal +from test_framework.util import assert_equal, connect_nodes from io import BytesIO - -ADDRESS = "tcp://127.0.0.1:28332" +from time import sleep def hash256_reversed(byte_str): return hash256(byte_str)[::-1] @@ -43,66 +42,64 @@ class ZMQTest (BitcoinTestFramework): self.skip_if_no_py3_zmq() self.skip_if_no_bitcoind_zmq() - def setup_nodes(self): + def run_test(self): import zmq + self.ctx = zmq.Context() + try: + self.test_basic() + self.test_reorg() + finally: + # Destroy the ZMQ context. + self.log.debug("Destroying ZMQ context") + self.ctx.destroy(linger=None) - # Initialize ZMQ context and socket. + def test_basic(self): # All messages are received in the same socket which means # that this test fails if the publishing order changes. # Note that the publishing order is not defined in the documentation and # is subject to change. - self.zmq_context = zmq.Context() - socket = self.zmq_context.socket(zmq.SUB) + import zmq + address = 'tcp://127.0.0.1:28332' + socket = self.ctx.socket(zmq.SUB) socket.set(zmq.RCVTIMEO, 60000) - socket.connect(ADDRESS) # Subscribe to all available topics. - self.hashblock = ZMQSubscriber(socket, b"hashblock") - self.hashtx = ZMQSubscriber(socket, b"hashtx") - self.rawblock = ZMQSubscriber(socket, b"rawblock") - self.rawtx = ZMQSubscriber(socket, b"rawtx") - - self.extra_args = [ - ["-zmqpub%s=%s" % (sub.topic.decode(), ADDRESS) for sub in [self.hashblock, self.hashtx, self.rawblock, self.rawtx]], - [], - ] - self.add_nodes(self.num_nodes, self.extra_args) - self.start_nodes() - self.import_deterministic_coinbase_privkeys() + hashblock = ZMQSubscriber(socket, b"hashblock") + hashtx = ZMQSubscriber(socket, b"hashtx") + rawblock = ZMQSubscriber(socket, b"rawblock") + rawtx = ZMQSubscriber(socket, b"rawtx") - def run_test(self): - try: - self._zmq_test() - finally: - # Destroy the ZMQ context. - self.log.debug("Destroying ZMQ context") - self.zmq_context.destroy(linger=None) + self.restart_node(0, ["-zmqpub%s=%s" % (sub.topic.decode(), address) for sub in [hashblock, hashtx, rawblock, rawtx]]) + connect_nodes(self.nodes[0], 1) + socket.connect(address) + # Relax so that the subscriber is ready before publishing zmq messages + sleep(0.2) - def _zmq_test(self): num_blocks = 5 self.log.info("Generate %(n)d blocks (and %(n)d coinbase txes)" % {"n": num_blocks}) genhashes = self.nodes[0].generatetoaddress(num_blocks, ADDRESS_BCRT1_UNSPENDABLE) + self.sync_all() for x in range(num_blocks): # Should receive the coinbase txid. - txid = self.hashtx.receive() + txid = hashtx.receive() # Should receive the coinbase raw transaction. - hex = self.rawtx.receive() + hex = rawtx.receive() tx = CTransaction() tx.deserialize(BytesIO(hex)) tx.calc_sha256() assert_equal(tx.hash, txid.hex()) # Should receive the generated block hash. - hash = self.hashblock.receive().hex() + hash = hashblock.receive().hex() assert_equal(genhashes[x], hash) # The block should only have the coinbase txid. assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) # Should receive the generated raw block. - block = self.rawblock.receive() + block = rawblock.receive() assert_equal(genhashes[x], hash256_reversed(block[:80]).hex()) if self.is_wallet_compiled(): @@ -111,23 +108,49 @@ class ZMQTest (BitcoinTestFramework): self.sync_all() # Should receive the broadcasted txid. - txid = self.hashtx.receive() + txid = hashtx.receive() assert_equal(payment_txid, txid.hex()) # Should receive the broadcasted raw transaction. - hex = self.rawtx.receive() + hex = rawtx.receive() assert_equal(payment_txid, hash256_reversed(hex).hex()) self.log.info("Test the getzmqnotifications RPC") assert_equal(self.nodes[0].getzmqnotifications(), [ - {"type": "pubhashblock", "address": ADDRESS, "hwm": 1000}, - {"type": "pubhashtx", "address": ADDRESS, "hwm": 1000}, - {"type": "pubrawblock", "address": ADDRESS, "hwm": 1000}, - {"type": "pubrawtx", "address": ADDRESS, "hwm": 1000}, + {"type": "pubhashblock", "address": address, "hwm": 1000}, + {"type": "pubhashtx", "address": address, "hwm": 1000}, + {"type": "pubrawblock", "address": address, "hwm": 1000}, + {"type": "pubrawtx", "address": address, "hwm": 1000}, ]) assert_equal(self.nodes[1].getzmqnotifications(), []) + def test_reorg(self): + import zmq + address = 'tcp://127.0.0.1:28333' + socket = self.ctx.socket(zmq.SUB) + socket.set(zmq.RCVTIMEO, 60000) + hashblock = ZMQSubscriber(socket, b'hashblock') + + # Should only notify the tip if a reorg occurs + self.restart_node(0, ['-zmqpub%s=%s' % (hashblock.topic.decode(), address)]) + socket.connect(address) + # Relax so that the subscriber is ready before publishing zmq messages + sleep(0.2) + + # Generate 1 block in nodes[0] and receive all notifications + self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + assert_equal(self.nodes[0].getbestblockhash(), hashblock.receive().hex()) + + # Generate 2 blocks in nodes[1] + self.nodes[1].generatetoaddress(2, ADDRESS_BCRT1_UNSPENDABLE) + + # nodes[0] will reorg chain after connecting back nodes[1] + connect_nodes(self.nodes[0], 1) + + # Should receive nodes[1] tip + assert_equal(self.nodes[1].getbestblockhash(), hashblock.receive().hex()) + if __name__ == '__main__': ZMQTest().main() diff --git a/test/functional/rpc_setban.py b/test/functional/rpc_setban.py index a1a8196557..423741fd27 100755 --- a/test/functional/rpc_setban.py +++ b/test/functional/rpc_setban.py @@ -26,7 +26,7 @@ class SetBanTests(BitcoinTestFramework): self.nodes[1].setban("127.0.0.1", "add") # Node 0 should not be able to reconnect - with self.nodes[1].assert_debug_log(expected_msgs=['dropped (banned)\n']): + with self.nodes[1].assert_debug_log(expected_msgs=['dropped (banned)\n'],timeout=5): self.restart_node(1, []) self.nodes[0].addnode("127.0.0.1:" + str(p2p_port(1)), "onetry") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index df027397d2..9667cf4ea4 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -307,7 +307,8 @@ class TestNode(): wait_until(self.is_node_stopped, timeout=timeout) @contextlib.contextmanager - def assert_debug_log(self, expected_msgs): + def assert_debug_log(self, expected_msgs, timeout=2): + time_end = time.time() + timeout debug_log = os.path.join(self.datadir, self.chain, 'debug.log') with open(debug_log, encoding='utf-8') as dl: dl.seek(0, 2) @@ -315,13 +316,21 @@ class TestNode(): try: yield finally: - with open(debug_log, encoding='utf-8') as dl: - dl.seek(prev_size) - log = dl.read() - print_log = " - " + "\n - ".join(log.splitlines()) - for expected_msg in expected_msgs: - if re.search(re.escape(expected_msg), log, flags=re.MULTILINE) is None: - self._raise_assertion_error('Expected message "{}" does not partially match log:\n\n{}\n\n'.format(expected_msg, print_log)) + while True: + found = True + with open(debug_log, encoding='utf-8') as dl: + dl.seek(prev_size) + log = dl.read() + print_log = " - " + "\n - ".join(log.splitlines()) + for expected_msg in expected_msgs: + if re.search(re.escape(expected_msg), log, flags=re.MULTILINE) is None: + found = False + if found: + return + if time.time() >= time_end: + break + time.sleep(0.05) + self._raise_assertion_error('Expected messages "{}" does not partially match log:\n\n{}\n\n'.format(str(expected_msgs), print_log)) @contextlib.contextmanager def assert_memory_usage_stable(self, *, increase_allowed=0.03): diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index be8d7714fb..be8d7714fb 100644..100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py |