diff options
162 files changed, 2407 insertions, 1509 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index b915cfabc7..801164c368 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -136,7 +136,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" task: - name: '[no depends, sanitizers: fuzzer,address,undefined] [focal]' + name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal diff --git a/Makefile.am b/Makefile.am index 7c4b971ea3..f6b824faaa 100644 --- a/Makefile.am +++ b/Makefile.am @@ -46,6 +46,9 @@ OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed DIST_CONTRIB = \ + $(top_srcdir)/test/sanitizer_suppressions/lsan \ + $(top_srcdir)/test/sanitizer_suppressions/tsan \ + $(top_srcdir)/test/sanitizer_suppressions/ubsan \ $(top_srcdir)/contrib/linearize/linearize-data.py \ $(top_srcdir)/contrib/linearize/linearize-hashes.py diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index de89a09d6a..e38cfe8eef 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -21,7 +21,6 @@ test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c test/lint/check-doc.py -test/lint/check-rpc-mappings.py . test/lint/lint-all.sh if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ -n "$CIRRUS_CRON" ]; then diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index a32de4a6b5..b7157c608d 100644 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -14,5 +14,5 @@ export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++ --with-boost-process" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC=clang CXX=clang++ --with-boost-process" export CCACHE_SIZE=200M diff --git a/configure.ac b/configure.ac index 1592ed352f..c16eae8ba1 100644 --- a/configure.ac +++ b/configure.ac @@ -427,6 +427,7 @@ if test "x$enable_werror" = "xyes"; then AX_CHECK_COMPILE_FLAG([-Werror=suggest-override],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=suggest-override"],,[[$CXXFLAG_WERROR]], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) AX_CHECK_COMPILE_FLAG([-Werror=unreachable-code-loop-increment],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Werror=mismatched-tags], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=mismatched-tags"], [], [$CXXFLAG_WERROR]) fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -813,6 +814,11 @@ AX_GCC_FUNC_ATTRIBUTE([dllimport]) if test x$use_glibc_compat != xno; then AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) + case $host in + powerpc64* | ppc64*) + AX_CHECK_LINK_FLAG([[-Wl,--no-tls-get-addr-optimize]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--no-tls-get-addr-optimize"]) + ;; + esac else AC_SEARCH_LIBS([clock_gettime],[rt]) fi @@ -1248,8 +1254,10 @@ if test "x$enable_fuzz" = "xyes"; then #include <cstdint> #include <cstddef> extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; } - /* unterminated comment to remove the main function ... - ]],[[]])]) + /* comment to remove the main function ... + ]],[[ + */ int not_main() { + ]])]) else BITCOIN_QT_INIT diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py index f105968515..06b15574a7 100755 --- a/contrib/gitian-build.py +++ b/contrib/gitian-build.py @@ -13,15 +13,16 @@ def setup(): programs = ['ruby', 'git', 'make', 'wget', 'curl'] if args.kvm: programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils'] - elif args.docker and not os.path.isfile('/lib/systemd/system/docker.service'): - dockers = ['docker.io', 'docker-ce'] - for i in dockers: - return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) - if return_code == 0: - break - if return_code != 0: - print('Cannot find any way to install Docker.', file=sys.stderr) - sys.exit(1) + elif args.docker: + if not os.path.isfile('/lib/systemd/system/docker.service'): + dockers = ['docker.io', 'docker-ce'] + for i in dockers: + return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) + if return_code == 0: + break + if return_code != 0: + print('Cannot find any way to install Docker.', file=sys.stderr) + sys.exit(1) else: programs += ['apt-cacher-ng', 'lxc', 'debootstrap'] subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 0e2c5be0fa..a0ff87b531 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -27,6 +27,12 @@ packages: # - aarch64-linux-gnu - "binutils-aarch64-linux-gnu" - "g++-8-aarch64-linux-gnu" +# - powerpc64-linux-gnu +- "binutils-powerpc64-linux-gnu" +- "g++-8-powerpc64-linux-gnu" +# - powerpc64le-linux-gnu +- "binutils-powerpc64le-linux-gnu" +- "g++-8-powerpc64le-linux-gnu" # - riscv64-linux-gnu - "binutils-riscv64-linux-gnu" - "g++-8-riscv64-linux-gnu" @@ -38,7 +44,7 @@ script: | set -e -o pipefail WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu" + HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu riscv64-linux-gnu" CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests" FAKETIME_HOST_PROGS="gcc g++" FAKETIME_PROGS="date ar ranlib nm" @@ -64,7 +70,7 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -78,7 +84,13 @@ script: | echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog} + if [ "${i:0:11}" = "powerpc64le" ]; then + echo "exec \"\$REAL\" -mcpu=power8 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} + elif [ "${i:0:9}" = "powerpc64" ]; then + echo "exec \"\$REAL\" -mcpu=970 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} + else + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} + fi chmod +x ${WRAP_DIR}/${i}-${prog} fi done @@ -118,7 +130,7 @@ script: | # Extract the git archive into a dir for each host and build for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - if [ "${i}" = "riscv64-linux-gnu" ]; then + if [ "${i}" = "riscv64-linux-gnu" ] || [ "${i}" = "powerpc64-linux-gnu" ] || [ "${i}" = "powerpc64le-linux-gnu" ]; then # Workaround for https://bugs.launchpad.net/ubuntu/+source/gcc-8-cross-ports/+bug/1853740 # TODO: remove this when no longer needed HOST_LDFLAGS="${HOST_LDFLAGS_BASE} -Wl,-z,noexecstack" diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 214ca9823d..6fcb21f729 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -31,7 +31,7 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 86f976f568..2a47e90e6e 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -63,7 +63,7 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -75,7 +75,7 @@ script: | echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index fc79745e69..1edd8b2e81 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -55,7 +55,7 @@ script: | echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } @@ -67,7 +67,7 @@ script: | echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done @@ -81,7 +81,7 @@ script: | echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog} + echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done diff --git a/contrib/message-capture/message-capture-docs.md b/contrib/message-capture/message-capture-docs.md new file mode 100644 index 0000000000..7301968461 --- /dev/null +++ b/contrib/message-capture/message-capture-docs.md @@ -0,0 +1,25 @@ +# Per-Peer Message Capture + +## Purpose + +This feature allows for message capture on a per-peer basis. It answers the simple question: "Can I see what messages my node is sending and receiving?" + +## Usage and Functionality + +* Run `bitcoind` with the `-capturemessages` option. +* Look in the `message_capture` folder in your datadir. + * Typically this will be `~/.bitcoin/message_capture`. + * See that there are many folders inside, one for each peer names with its IP address and port. + * Inside each peer's folder there are two `.dat` files: one is for received messages (`msgs_recv.dat`) and the other is for sent messages (`msgs_sent.dat`). +* Run `contrib/message-capture/message-capture-parser.py` with the proper arguments. + * See the `-h` option for help. + * To see all messages, both sent and received, for all peers use: + ``` + ./contrib/message-capture/message-capture-parser.py -o out.json \ + ~/.bitcoin/message_capture/**/*.dat + ``` + * Note: The messages in the given `.dat` files will be interleaved in chronological order. So, giving both received and sent `.dat` files (as above with `*.dat`) will result in all messages being interleaved in chronological order. + * If an output file is not provided (i.e. the `-o` option is not used), then the output prints to `stdout`. +* View the resulting output. + * The output file is `JSON` formatted. + * Suggestion: use `jq` to view the output, with `jq . out.json` diff --git a/contrib/message-capture/message-capture-parser.py b/contrib/message-capture/message-capture-parser.py new file mode 100755 index 0000000000..9988478f1b --- /dev/null +++ b/contrib/message-capture/message-capture-parser.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Parse message capture binary files. To be used in conjunction with -capturemessages.""" + +import argparse +import os +import shutil +import sys +from io import BytesIO +import json +from pathlib import Path +from typing import Any, List, Optional + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) + +from test_framework.messages import ser_uint256 # noqa: E402 +from test_framework.p2p import MESSAGEMAP # noqa: E402 + +TIME_SIZE = 8 +LENGTH_SIZE = 4 +MSGTYPE_SIZE = 12 + +# The test framework classes stores hashes as large ints in many cases. +# These are variables of type uint256 in core. +# There isn't a way to distinguish between a large int and a large int that is actually a blob of bytes. +# As such, they are itemized here. +# Any variables with these names that are of type int are actually uint256 variables. +# (These can be easily found by looking for calls to deser_uint256, deser_uint256_vector, and uint256_from_str in messages.py) +HASH_INTS = [ + "blockhash", + "block_hash", + "hash", + "hashMerkleRoot", + "hashPrevBlock", + "hashstop", + "prev_header", + "sha256", + "stop_hash", +] + +HASH_INT_VECTORS = [ + "hashes", + "headers", + "vHave", + "vHash", +] + + +class ProgressBar: + def __init__(self, total: float): + self.total = total + self.running = 0 + + def set_progress(self, progress: float): + cols = shutil.get_terminal_size()[0] + if cols <= 12: + return + max_blocks = cols - 9 + num_blocks = int(max_blocks * progress) + print('\r[ {}{} ] {:3.0f}%' + .format('#' * num_blocks, + ' ' * (max_blocks - num_blocks), + progress * 100), + end ='') + + def update(self, more: float): + self.running += more + self.set_progress(self.running / self.total) + + +def to_jsonable(obj: Any) -> Any: + if hasattr(obj, "__dict__"): + return obj.__dict__ + elif hasattr(obj, "__slots__"): + ret = {} # type: Any + for slot in obj.__slots__: + val = getattr(obj, slot, None) + if slot in HASH_INTS and isinstance(val, int): + ret[slot] = ser_uint256(val).hex() + elif slot in HASH_INT_VECTORS and isinstance(val[0], int): + ret[slot] = [ser_uint256(a).hex() for a in val] + else: + ret[slot] = to_jsonable(val) + return ret + elif isinstance(obj, list): + return [to_jsonable(a) for a in obj] + elif isinstance(obj, bytes): + return obj.hex() + else: + return obj + + +def process_file(path: str, messages: List[Any], recv: bool, progress_bar: Optional[ProgressBar]) -> None: + with open(path, 'rb') as f_in: + if progress_bar: + bytes_read = 0 + + while True: + if progress_bar: + # Update progress bar + diff = f_in.tell() - bytes_read - 1 + progress_bar.update(diff) + bytes_read = f_in.tell() - 1 + + # Read the Header + tmp_header_raw = f_in.read(TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + if not tmp_header_raw: + break + tmp_header = BytesIO(tmp_header_raw) + time = int.from_bytes(tmp_header.read(TIME_SIZE), "little") # type: int + msgtype = tmp_header.read(MSGTYPE_SIZE).split(b'\x00', 1)[0] # type: bytes + length = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") # type: int + + # Start converting the message to a dictionary + msg_dict = {} + msg_dict["direction"] = "recv" if recv else "sent" + msg_dict["time"] = time + msg_dict["size"] = length # "size" is less readable here, but more readable in the output + + msg_ser = BytesIO(f_in.read(length)) + + # Determine message type + if msgtype not in MESSAGEMAP: + # Unrecognized message type + try: + msgtype_tmp = msgtype.decode() + if not msgtype_tmp.isprintable(): + raise UnicodeDecodeError + msg_dict["msgtype"] = msgtype_tmp + except UnicodeDecodeError: + msg_dict["msgtype"] = "UNREADABLE" + msg_dict["body"] = msg_ser.read().hex() + msg_dict["error"] = "Unrecognized message type." + messages.append(msg_dict) + print(f"WARNING - Unrecognized message type {msgtype} in {path}", file=sys.stderr) + continue + + # Deserialize the message + msg = MESSAGEMAP[msgtype]() + msg_dict["msgtype"] = msgtype.decode() + + try: + msg.deserialize(msg_ser) + except KeyboardInterrupt: + raise + except Exception: + # Unable to deserialize message body + msg_ser.seek(0, os.SEEK_SET) + msg_dict["body"] = msg_ser.read().hex() + msg_dict["error"] = "Unable to deserialize message." + messages.append(msg_dict) + print(f"WARNING - Unable to deserialize message in {path}", file=sys.stderr) + continue + + # Convert body of message into a jsonable object + if length: + msg_dict["body"] = to_jsonable(msg) + messages.append(msg_dict) + + if progress_bar: + # Update the progress bar to the end of the current file + # in case we exited the loop early + f_in.seek(0, os.SEEK_END) # Go to end of file + diff = f_in.tell() - bytes_read - 1 + progress_bar.update(diff) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, + epilog="EXAMPLE \n\t{0} -o out.json <data-dir>/message_capture/**/*.dat".format(sys.argv[0]), + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + "capturepaths", + nargs='+', + help="binary message capture files to parse.") + parser.add_argument( + "-o", "--output", + help="output file. If unset print to stdout") + parser.add_argument( + "-n", "--no-progress-bar", + action='store_true', + help="disable the progress bar. Automatically set if the output is not a terminal") + args = parser.parse_args() + capturepaths = [Path.cwd() / Path(capturepath) for capturepath in args.capturepaths] + output = Path.cwd() / Path(args.output) if args.output else False + use_progress_bar = (not args.no_progress_bar) and sys.stdout.isatty() + + messages = [] # type: List[Any] + if use_progress_bar: + total_size = sum(capture.stat().st_size for capture in capturepaths) + progress_bar = ProgressBar(total_size) + else: + progress_bar = None + + for capture in capturepaths: + process_file(str(capture), messages, "recv" in capture.stem, progress_bar) + + messages.sort(key=lambda msg: msg['time']) + + if use_progress_bar: + progress_bar.set_progress(1) + + jsonrep = json.dumps(messages) + if output: + with open(str(output), 'w+', encoding="utf8") as f_out: + f_out.write(jsonrep) + else: + print(jsonrep) + +if __name__ == "__main__": + main() diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md index 4209fdb364..c50d4bef71 100644 --- a/contrib/verifybinaries/README.md +++ b/contrib/verifybinaries/README.md @@ -21,21 +21,21 @@ The script returns 0 if everything passes the checks. It returns 1 if either the ```sh -./verify.sh bitcoin-core-0.11.2 -./verify.sh bitcoin-core-0.12.0 -./verify.sh bitcoin-core-0.13.0-rc3 +./verify.py bitcoin-core-0.11.2 +./verify.py bitcoin-core-0.12.0 +./verify.py bitcoin-core-0.13.0-rc3 ``` If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: ```sh -./verify.sh bitcoin-core-0.11.2-osx -./verify.sh 0.12.0-linux -./verify.sh bitcoin-core-0.13.0-rc3-win64 +./verify.py bitcoin-core-0.11.2-osx +./verify.py 0.12.0-linux +./verify.py bitcoin-core-0.13.0-rc3-win64 ``` If you do not want to keep the downloaded binaries, specify anything as the second parameter. ```sh -./verify.sh bitcoin-core-0.13.0 delete +./verify.py bitcoin-core-0.13.0 delete ``` diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py new file mode 100755 index 0000000000..b97ad11b8e --- /dev/null +++ b/contrib/verifybinaries/verify.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Script for verifying Bitoin Core release binaries + +This script attempts to download the signature file SHA256SUMS.asc from +bitcoincore.org and bitcoin.org and compares them. +It first checks if the signature passes, and then downloads the files +specified in the file, and checks if the hashes of these files match those +that are specified in the signature file. +The script returns 0 if everything passes the checks. It returns 1 if either +the signature check or the hash check doesn't pass. If an error occurs the +return value is >= 2. +""" +from hashlib import sha256 +import os +import subprocess +import sys +from textwrap import indent + +WORKINGDIR = "/tmp/bitcoin_verify_binaries" +HASHFILE = "hashes.tmp" +HOST1 = "https://bitcoincore.org" +HOST2 = "https://bitcoin.org" +VERSIONPREFIX = "bitcoin-core-" +SIGNATUREFILENAME = "SHA256SUMS.asc" + + +def parse_version_string(version_str): + if version_str.startswith(VERSIONPREFIX): # remove version prefix + version_str = version_str[len(VERSIONPREFIX):] + + parts = version_str.split('-') + version_base = parts[0] + version_rc = "" + version_os = "" + if len(parts) == 2: # "<version>-rcN" or "version-platform" + if "rc" in parts[1]: + version_rc = parts[1] + else: + version_os = parts[1] + elif len(parts) == 3: # "<version>-rcN-platform" + version_rc = parts[1] + version_os = parts[2] + + return version_base, version_rc, version_os + + +def download_with_wget(remote_file, local_file=None): + if local_file: + wget_args = ['wget', '-O', local_file, remote_file] + else: + # use timestamping mechanism if local filename is not explicitely set + wget_args = ['wget', '-N', remote_file] + + result = subprocess.run(wget_args, + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode == 0, result.stdout.decode().rstrip() + + +def files_are_equal(filename1, filename2): + with open(filename1, 'rb') as file1: + contents1 = file1.read() + with open(filename2, 'rb') as file2: + contents2 = file2.read() + return contents1 == contents2 + + +def verify_with_gpg(signature_filename, output_filename): + result = subprocess.run(['gpg', '--yes', '--decrypt', '--output', + output_filename, signature_filename], + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode, result.stdout.decode().rstrip() + + +def remove_files(filenames): + for filename in filenames: + os.remove(filename) + + +def main(args): + # sanity check + if len(args) < 1: + print("Error: need to specify a version on the command line") + return 3 + + # determine remote dir dependend on provided version string + version_base, version_rc, os_filter = parse_version_string(args[0]) + remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/" + if version_rc: + remote_dir += f"test.{version_rc}/" + remote_sigfile = remote_dir + SIGNATUREFILENAME + + # create working directory + os.makedirs(WORKINGDIR, exist_ok=True) + os.chdir(WORKINGDIR) + + # fetch first signature file + sigfile1 = SIGNATUREFILENAME + success, output = download_with_wget(HOST1 + remote_sigfile, sigfile1) + if not success: + print("Error: couldn't fetch signature file. " + "Have you specified the version number in the following format?") + print(f"[{VERSIONPREFIX}]<version>[-rc[0-9]][-platform] " + f"(example: {VERSIONPREFIX}0.21.0-rc3-osx)") + print("wget output:") + print(indent(output, '\t')) + return 4 + + # fetch second signature file + sigfile2 = SIGNATUREFILENAME + ".2" + success, output = download_with_wget(HOST2 + remote_sigfile, sigfile2) + if not success: + print("bitcoin.org failed to provide signature file, " + "but bitcoincore.org did?") + print("wget output:") + print(indent(output, '\t')) + remove_files([sigfile1]) + return 5 + + # ensure that both signature files are equal + if not files_are_equal(sigfile1, sigfile2): + print("bitcoin.org and bitcoincore.org signature files were not equal?") + print(f"See files {WORKINGDIR}/{sigfile1} and {WORKINGDIR}/{sigfile2}") + return 6 + + # check signature and extract data into file + retval, output = verify_with_gpg(sigfile1, HASHFILE) + if retval != 0: + if retval == 1: + print("Bad signature.") + elif retval == 2: + print("gpg error. Do you have the Bitcoin Core binary release " + "signing key installed?") + print("gpg output:") + print(indent(output, '\t')) + remove_files([sigfile1, sigfile2, HASHFILE]) + return 1 + + # extract hashes/filenames of binaries to verify from hash file; + # each line has the following format: "<hash> <binary_filename>" + with open(HASHFILE, 'r', encoding='utf8') as hash_file: + hashes_to_verify = [ + line.split()[:2] for line in hash_file if os_filter in line] + remove_files([HASHFILE]) + if not hashes_to_verify: + print("error: no files matched the platform specified") + return 7 + + # download binaries + for _, binary_filename in hashes_to_verify: + print(f"Downloading {binary_filename}") + download_with_wget(HOST1 + remote_dir + binary_filename) + + # verify hashes + offending_files = [] + for hash_expected, binary_filename in hashes_to_verify: + with open(binary_filename, 'rb') as binary_file: + hash_calculated = sha256(binary_file.read()).hexdigest() + if hash_calculated != hash_expected: + offending_files.append(binary_filename) + if offending_files: + print("Hashes don't match.") + print("Offending files:") + print('\n'.join(offending_files)) + return 1 + verified_binaries = [entry[1] for entry in hashes_to_verify] + + # clean up files if desired + if len(args) >= 2: + print("Clean up the binaries") + remove_files([sigfile1, sigfile2] + verified_binaries) + else: + print(f"Keep the binaries in {WORKINGDIR}") + + print("Verified hashes of") + print('\n'.join(verified_binaries)) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/contrib/verifybinaries/verify.sh b/contrib/verifybinaries/verify.sh deleted file mode 100755 index 4296998631..0000000000 --- a/contrib/verifybinaries/verify.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -### This script attempts to download the signature file SHA256SUMS.asc from -### bitcoincore.org and bitcoin.org and compares them. -### It first checks if the signature passes, and then downloads the files specified in -### the file, and checks if the hashes of these files match those that are specified -### in the signature file. -### The script returns 0 if everything passes the checks. It returns 1 if either the -### signature check or the hash check doesn't pass. If an error occurs the return value is 2 - -export LC_ALL=C -function clean_up { - for file in "$@" - do - rm "$file" 2> /dev/null - done -} - -WORKINGDIR="/tmp/bitcoin_verify_binaries" -TMPFILE="hashes.tmp" - -SIGNATUREFILENAME="SHA256SUMS.asc" -RCSUBDIR="test" -HOST1="https://bitcoincore.org" -HOST2="https://bitcoin.org" -BASEDIR="/bin/" -VERSIONPREFIX="bitcoin-core-" -RCVERSIONSTRING="rc" - -if [ ! -d "$WORKINGDIR" ]; then - mkdir "$WORKINGDIR" -fi - -cd "$WORKINGDIR" || exit 1 - -#test if a version number has been passed as an argument -if [ -n "$1" ]; then - #let's also check if the version number includes the prefix 'bitcoin-', - # and add this prefix if it doesn't - if [[ $1 == "$VERSIONPREFIX"* ]]; then - VERSION="$1" - else - VERSION="$VERSIONPREFIX$1" - fi - - STRIPPEDLAST="${VERSION%-*}" - - #now let's see if the version string contains "rc" or a platform name (e.g. "osx") - if [[ "$STRIPPEDLAST-" == "$VERSIONPREFIX" ]]; then - BASEDIR="$BASEDIR$VERSION/" - else - # let's examine the last part to see if it's rc and/or platform name - STRIPPEDNEXTTOLAST="${STRIPPEDLAST%-*}" - if [[ "$STRIPPEDNEXTTOLAST-" == "$VERSIONPREFIX" ]]; then - - LASTSUFFIX="${VERSION##*-}" - VERSION="$STRIPPEDLAST" - - if [[ $LASTSUFFIX == *"$RCVERSIONSTRING"* ]]; then - RCVERSION="$LASTSUFFIX" - else - PLATFORM="$LASTSUFFIX" - fi - - else - RCVERSION="${STRIPPEDLAST##*-}" - PLATFORM="${VERSION##*-}" - - VERSION="$STRIPPEDNEXTTOLAST" - fi - - BASEDIR="$BASEDIR$VERSION/" - if [[ $RCVERSION == *"$RCVERSIONSTRING"* ]]; then - BASEDIR="$BASEDIR$RCSUBDIR.$RCVERSION/" - fi - fi -else - echo "Error: need to specify a version on the command line" - exit 2 -fi - -if ! WGETOUT=$(wget -N "$HOST1$BASEDIR$SIGNATUREFILENAME" 2>&1); then - echo "Error: couldn't fetch signature file. Have you specified the version number in the following format?" - # shellcheck disable=SC1087 - echo "[$VERSIONPREFIX]<version>-[$RCVERSIONSTRING[0-9]] (example: ${VERSIONPREFIX}0.10.4-${RCVERSIONSTRING}1)" - echo "wget output:" - # shellcheck disable=SC2001 - echo "$WGETOUT"|sed 's/^/\t/g' - exit 2 -fi - -if ! WGETOUT=$(wget -N -O "$SIGNATUREFILENAME.2" "$HOST2$BASEDIR$SIGNATUREFILENAME" 2>&1); then - echo "bitcoin.org failed to provide signature file, but bitcoincore.org did?" - echo "wget output:" - # shellcheck disable=SC2001 - echo "$WGETOUT"|sed 's/^/\t/g' - clean_up $SIGNATUREFILENAME - exit 3 -fi - -SIGFILEDIFFS="$(diff $SIGNATUREFILENAME $SIGNATUREFILENAME.2)" -if [ "$SIGFILEDIFFS" != "" ]; then - echo "bitcoin.org and bitcoincore.org signature files were not equal?" - clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2 - exit 4 -fi - -#then we check it -GPGOUT=$(gpg --yes --decrypt --output "$TMPFILE" "$SIGNATUREFILENAME" 2>&1) - -#return value 0: good signature -#return value 1: bad signature -#return value 2: gpg error - -RET="$?" -if [ $RET -ne 0 ]; then - if [ $RET -eq 1 ]; then - #and notify the user if it's bad - echo "Bad signature." - elif [ $RET -eq 2 ]; then - #or if a gpg error has occurred - echo "gpg error. Do you have the Bitcoin Core binary release signing key installed?" - fi - - echo "gpg output:" - # shellcheck disable=SC2001 - echo "$GPGOUT"|sed 's/^/\t/g' - clean_up $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE - exit "$RET" -fi - -if [ -n "$PLATFORM" ]; then - grep $PLATFORM $TMPFILE > "$TMPFILE-plat" - TMPFILESIZE=$(stat -c%s "$TMPFILE-plat") - if [ $TMPFILESIZE -eq 0 ]; then - echo "error: no files matched the platform specified" && exit 3 - fi - mv "$TMPFILE-plat" $TMPFILE -fi - -#here we extract the filenames from the signature file -FILES=$(awk '{print $2}' "$TMPFILE") - -#and download these one by one -for file in $FILES -do - echo "Downloading $file" - wget --quiet -N "$HOST1$BASEDIR$file" -done - -#check hashes -DIFF=$(diff <(sha256sum $FILES) "$TMPFILE") - -if [ $? -eq 1 ]; then - echo "Hashes don't match." - echo "Offending files:" - echo "$DIFF"|grep "^<"|awk '{print "\t"$3}' - exit 1 -elif [ $? -gt 1 ]; then - echo "Error executing 'diff'" - exit 2 -fi - -if [ -n "$2" ]; then - echo "Clean up the binaries" - clean_up $FILES $SIGNATUREFILENAME $SIGNATUREFILENAME.2 $TMPFILE -else - echo "Keep the binaries in $WORKINGDIR" - clean_up $TMPFILE -fi - -echo -e "Verified hashes of \n$FILES" - -exit 0 diff --git a/depends/Makefile b/depends/Makefile index 596a46d4a2..016b73a5cb 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -265,7 +265,7 @@ install: check-packages $(host_prefix)/share/config.site download-one: check-sources $(all_sources) download-osx: - @$(MAKE) -s HOST=x86_64-apple-darwin14 download-one + @$(MAKE) -s HOST=x86_64-apple-darwin download-one download-linux: @$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one download-win: diff --git a/doc/release-notes.md b/doc/release-notes.md index 8f1e03e16b..0f248494c7 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -93,6 +93,10 @@ Tools and Utilities Wallet ------ +- A new `listdescriptors` RPC is available to inspect the contents of descriptor-enabled wallets. + The RPC returns public versions of all imported descriptors, including their timestamp and flags. + For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) + GUI changes ----------- diff --git a/doc/release-process.md b/doc/release-process.md index 92845bcc82..84b208a0d8 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -5,7 +5,7 @@ Release Process ### Before every release candidate -* Update translations (ping wumpus on IRC) see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). +* Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). * Update manpages, see [gen-manpages.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagessh). * Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). @@ -52,6 +52,13 @@ Release Process - Merge the release notes from the wiki into the branch. - Ensure the "Needs release note" label is removed from all relevant pull requests and issues. +#### Tagging a release (candidate) + +To tag the version (or release candidate) in git, use the `make-tag.py` script from [bitcoin-maintainer-tools](https://github.com/bitcoin-core/bitcoin-maintainer-tools). From the root of the repository run: + + ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 0.20.0) + +This will perform a few last-minute consistency checks in the build system files, and if they pass, create a signed tag. ## Building @@ -73,21 +80,12 @@ Open a draft of the release notes for collaborative editing at https://github.co For the period during which the notes are being edited on the wiki, the version on the branch should be wiped and replaced with a link to the wiki which should be used for all announcements until `-final`. -Write the release notes. `git shortlog` helps a lot, for example: - - git shortlog --no-merges v(current version, e.g. 0.19.2)..v(new version, e.g. 0.20.0) - -(or ping @wumpus on IRC, he has specific tooling to generate the list of merged pulls -and sort them into categories based on labels). +Generate the change log. As this is a huge amount of work to do manually, there is the `list-pulls` script to do a pre-sorting step based on github PR metadata. See the [documentation in the README.md](https://github.com/bitcoin-core/bitcoin-maintainer-tools/blob/master/README.md#list-pulls). Generate list of authors: git log --format='- %aN' v(current version, e.g. 0.20.0)..v(new version, e.g. 0.20.1) | sort -fiu -Tag the version (or release candidate) in git: - - git tag -s v(new version, e.g. 0.20.0) - ### Setup and perform Gitian builds If you're using the automated script (found in [contrib/gitian-build.py](/contrib/gitian-build.py)), then at this point you should run it with the "--build" command. Otherwise ignore this. @@ -326,6 +324,18 @@ bitcoin.org (see below for bitcoin.org update instructions). - bitcoincore.org RPC documentation update + - Install [golang](https://golang.org/doc/install) + + - Install the new Bitcoin Core release + + - Run bitcoind on regtest + + - Clone the [bitcoincore.org repository](https://github.com/bitcoin-core/bitcoincore.org) + + - Run: `go run generate.go` while being in `contrib/doc-gen` folder, and with bitcoin-cli in PATH + + - Add the generated files to git + - Update packaging repo - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.bitcoin-qt/pull/2 diff --git a/doc/tor.md b/doc/tor.md index 1ba7137b8e..8a2aef2d07 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -23,14 +23,19 @@ outgoing connections, but more is possible. -proxy=ip:port Set the proxy server. If SOCKS5 is selected (default), this proxy server will be used to try to reach .onion addresses as well. + You need to use -noonion or -onion=0 to explicitly disable + outbound access to onion services. -onion=ip:port Set the proxy server to use for Tor onion services. You do not - need to set this if it's the same as -proxy. You can use -noonion + need to set this if it's the same as -proxy. You can use -onion=0 to explicitly disable access to onion services. + Note: Only the -proxy option sets the proxy for DNS requests; + with -onion they will not route over Tor, so use -proxy if you + have privacy concerns. -listen When using -proxy, listening is disabled by default. If you want - to run an onion service (see next section), you'll need to enable - it explicitly. + to manually configure an onion service (see section 3), you'll + need to enable it explicitly. -connect=X When behind a Tor proxy, you can specify .onion addresses instead -addnode=X of IP addresses or hostnames in these parameters. It requires @@ -40,19 +45,112 @@ outgoing connections, but more is possible. -onlynet=onion Make outgoing connections only to .onion addresses. Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple network types, e.g. - ipv4, ipv6, or onion. + ipv4, ipv6 or onion. If you use this option with values other + than onion you *cannot* disable onion connections; outgoing onion + connections will be enabled when you use -proxy or -onion. Use + -noonion or -onion=0 if you want to be sure there are no outbound + onion connections over the default proxy or your defined -proxy. In a typical situation, this suffices to run behind a Tor proxy: ./bitcoind -proxy=127.0.0.1:9050 +## 2. Automatically create a Bitcoin Core onion service -## 2. Manually create a Bitcoin Core onion service +Bitcoin Core makes use of Tor's control socket API to create and destroy +ephemeral onion services programmatically. This means that if Tor is running and +proper authentication has been configured, Bitcoin Core automatically creates an +onion service to listen on. The goal is to increase the number of available +onion nodes. -If you configure your Tor system accordingly, it is possible to make your node also -reachable from the Tor network. Add these lines to your /etc/tor/torrc (or equivalent -config file): *Needed for Tor version 0.2.7.0 and older versions of Tor only. For newer -versions of Tor see [Section 3](#3-automatically-listen-on-tor).* +This feature is enabled by default if Bitcoin Core is listening (`-listen`) and +it requires a Tor connection to work. It can be explicitly disabled with +`-listenonion=0`. If it is not disabled, it can be configured using the +`-torcontrol` and `-torpassword` settings. + +To see verbose Tor information in the bitcoind debug log, pass `-debug=tor`. + +### Control Port + +You may need to set up the Tor Control Port. On Linux distributions there may be +some or all of the following settings in `/etc/tor/torrc`, generally commented +out by default (if not, add them): + +``` +ControlPort 9051 +CookieAuthentication 1 +CookieAuthFileGroupReadable 1 +``` + +Add or uncomment those, save, and restart Tor (usually `systemctl restart tor` +or `sudo systemctl restart tor` on most systemd-based systems, including recent +Debian and Ubuntu, or just restart the computer). + +On some systems (such as Arch Linux), you may also need to add the following +line: + +``` +DataDirectoryGroupReadable 1 +``` + +### Authentication + +Connecting to Tor's control socket API requires one of two authentication +methods to be configured: cookie authentication or bitcoind's `-torpassword` +configuration option. + +#### Cookie authentication + +For cookie authentication, the user running bitcoind must have read access to +the `CookieAuthFile` specified in the Tor configuration. In some cases this is +preconfigured and the creation of an onion service is automatic. Don't forget to +use the `-debug=tor` bitcoind configuration option to enable Tor debug logging. + +If a permissions problem is seen in the debug log, e.g. `tor: Authentication +cookie /run/tor/control.authcookie could not be opened (check permissions)`, it +can be resolved by adding both the user running Tor and the user running +bitcoind to the same Tor group and setting permissions appropriately. + +On Debian-derived systems, the Tor group will likely be `debian-tor` and one way +to verify could be to list the groups and grep for a "tor" group name: + +``` +getent group | cut -d: -f1 | grep -i tor +``` + +You can also check the group of the cookie file. On most Linux systems, the Tor +auth cookie will usually be `/run/tor/control.authcookie`: + +``` +stat -c '%G' /run/tor/control.authcookie +``` + +Once you have determined the `${TORGROUP}` and selected the `${USER}` that will +run bitcoind, run this as root: + +``` +usermod -a -G ${TORGROUP} ${USER} +``` + +Then restart the computer (or log out) and log in as the `${USER}` that will run +bitcoind. + +#### `torpassword` authentication + +For the `-torpassword=password` option, the password is the clear text form that +was used when generating the hashed password for the `HashedControlPassword` +option in the Tor configuration file. + +The hashed password can be obtained with the command `tor --hash-password +password` (refer to the [Tor Dev +Manual](https://2019.www.torproject.org/docs/tor-manual.html.en) for more +details). + + +## 3. Manually create a Bitcoin Core onion service + +You can also manually configure your node to be reachable from the Tor network. +Add these lines to your `/etc/tor/torrc` (or equivalent config file): HiddenServiceDir /var/lib/tor/bitcoin-service/ HiddenServicePort 8333 127.0.0.1:8334 @@ -106,44 +204,10 @@ for normal IPv4/IPv6 communication, use: ./bitcoind -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover -## 3. Automatically create a Bitcoin Core onion service - -Starting with Tor version 0.2.7.1 it is possible, through Tor's control socket -API, to create and destroy 'ephemeral' onion services programmatically. -Bitcoin Core has been updated to make use of this. - -This means that if Tor is running (and proper authentication has been configured), -Bitcoin Core automatically creates an onion service to listen on. This will positively -affect the number of available .onion nodes. - -This new feature is enabled by default if Bitcoin Core is listening (`-listen`), and -requires a Tor connection to work. It can be explicitly disabled with `-listenonion=0` -and, if not disabled, configured using the `-torcontrol` and `-torpassword` settings. -To show verbose debugging information, pass `-debug=tor`. - -Connecting to Tor's control socket API requires one of two authentication methods to be -configured. It also requires the control socket to be enabled, e.g. put `ControlPort 9051` -in `torrc` config file. For cookie authentication the user running bitcoind must have read -access to the `CookieAuthFile` specified in Tor configuration. In some cases this is -preconfigured and the creation of an onion service is automatic. If permission problems -are seen with `-debug=tor` they can be resolved by adding both the user running Tor and -the user running bitcoind to the same group and setting permissions appropriately. On -Debian-based systems the user running bitcoind can be added to the debian-tor group, -which has the appropriate permissions. Before starting bitcoind you will need to re-login -to allow debian-tor group to be applied. Otherwise you will see the following notice: "tor: -Authentication cookie /run/tor/control.authcookie could not be opened (check permissions)" -on debug.log. - -An alternative authentication method is the use -of the `-torpassword=password` option. The `password` is the clear text form that -was used when generating the hashed password for the `HashedControlPassword` option -in the tor configuration file. The hashed password can be obtained with the command -`tor --hash-password password` (read the tor manual for more details). - ## 4. Privacy recommendations -- Do not add anything but Bitcoin Core ports to the onion service created in section 2. +- Do not add anything but Bitcoin Core ports to the onion service created in section 3. If you run a web service too, create a new onion service for that. Otherwise it is trivial to link them, which may reduce privacy. Onion - services created automatically (as in section 3) always have only one port + services created automatically (as in section 2) always have only one port open. diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 4e858979fe..75fe68fcd1 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -16,6 +16,7 @@ libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAG libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ + test/fuzz/util.cpp \ $(TEST_FUZZ_H) LIBTEST_FUZZ += $(LIBBITCOIN_SERVER) diff --git a/src/addrman.cpp b/src/addrman.cpp index ed7fccc0ff..f91121f156 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -9,6 +9,8 @@ #include <logging.h> #include <serialize.h> +#include <cmath> + int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); diff --git a/src/bench/bench.h b/src/bench/bench.h index bafc7f8716..22f06d8cb8 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <util/macros.h> + #include <chrono> #include <functional> #include <map> @@ -12,8 +14,6 @@ #include <vector> #include <bench/nanobench.h> -#include <boost/preprocessor/cat.hpp> -#include <boost/preprocessor/stringize.hpp> /* * Usage: @@ -56,8 +56,8 @@ public: static void RunAll(const Args& args); }; } -// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo"); +// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); #define BENCHMARK(n) \ - benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n); + benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index ffa772d8c1..d7b8c1badc 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -10,8 +10,6 @@ #include <random.h> #include <util/system.h> -#include <boost/thread/thread.hpp> - #include <vector> static const size_t BATCHES = 101; @@ -44,12 +42,9 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) void swap(PrevectorJob& x){p.swap(x.p);}; }; CCheckQueue<PrevectorJob> queue {QUEUE_BATCH_SIZE}; - boost::thread_group tg; // The main thread should be counted to prevent thread oversubscription, and // to decrease the variance of benchmark results. - for (auto x = 0; x < GetNumCores() - 1; ++x) { - tg.create_thread([&]{queue.Thread();}); - } + queue.StartWorkerThreads(GetNumCores() - 1); // create all the data once, then submit copies in the benchmark. FastRandomContext insecure_rand(true); @@ -70,8 +65,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) // it is done explicitly here for clarity control.Wait(); }); - tg.interrupt_all(); - tg.join_all(); + queue.StopWorkerThreads(); ECC_Stop(); } BENCHMARK(CCheckQueueSpeedPrevectorJob); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index e50c4476bb..3abfbfd784 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -42,20 +42,19 @@ static void CoinSelection(benchmark::Bench& bench) } addCoin(3 * COIN, wallet, wtxs); - // Create groups - std::vector<OutputGroup> groups; + // Create coins + std::vector<COutput> coins; for (const auto& wtx : wtxs) { - COutput output(wtx.get(), 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); - groups.emplace_back(output.GetInputCoin(), 6, false, 0, 0); + coins.emplace_back(wtx.get(), 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); } const CoinEligibilityFilter filter_standard(1, 6, 0); - const CoinSelectionParams coin_selection_params(true, 34, 148, CFeeRate(0), 0); + const CoinSelectionParams coin_selection_params(true, 34, 148, CFeeRate(0), 0, false); bench.run([&] { std::set<CInputCoin> setCoinsRet; CAmount nValueRet; bool bnb_used; - bool success = wallet.SelectCoinsMinConf(1003 * COIN, filter_standard, groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used); + bool success = wallet.SelectCoinsMinConf(1003 * COIN, filter_standard, coins, setCoinsRet, nValueRet, coin_selection_params, bnb_used); assert(success); assert(nValueRet == 1003 * COIN); assert(setCoinsRet.size() == 2); @@ -75,7 +74,8 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup> tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; std::unique_ptr<CWalletTx> wtx = MakeUnique<CWalletTx>(&testWallet, MakeTransactionRef(std::move(tx))); - set.emplace_back(COutput(wtx.get(), nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0); + set.emplace_back(); + set.back().Insert(COutput(wtx.get(), nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0, false); wtxn.emplace_back(std::move(wtx)); } // Copied from src/wallet/test/coinselector_tests.cpp diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index dcd0e10285..f4fabedab6 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -76,7 +76,7 @@ static void PrevectorDeserialize(benchmark::Bench& bench) for (auto x = 0; x < 1000; ++x) { s0 >> t1; } - s0.Init(SER_NETWORK, 0); + s0.Rewind(); }); } diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 94043a6b45..fa41208a31 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -21,6 +21,7 @@ #include <util/url.h> #include <algorithm> +#include <cmath> #include <functional> #include <memory> #include <stdio.h> @@ -57,7 +58,7 @@ static void SetupCliArgs(ArgsManager& argsman) argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-generate", strprintf("Generate blocks immediately, equivalent to RPC generatenewaddress followed by RPC generatetoaddress. Optional positional integer arguments are number of blocks to generate (default: %s) and maximum iterations to try (default: %s), equivalent to RPC generatetoaddress nblocks and maxtries arguments. Example: bitcoin-cli -generate 4 1000", DEFAULT_NBLOCKS, DEFAULT_MAX_TRIES), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-generate", strprintf("Generate blocks immediately, equivalent to RPC getnewaddress followed by RPC generatetoaddress. Optional positional integer arguments are number of blocks to generate (default: %s) and maximum iterations to try (default: %s), equivalent to RPC generatetoaddress nblocks and maxtries arguments. Example: bitcoin-cli -generate 4 1000", DEFAULT_NBLOCKS, DEFAULT_MAX_TRIES), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-getinfo", "Get general information from the remote server. Note that unlike server-side RPC calls, the results of -getinfo is the result of multiple non-atomic requests. Some entries in the result may represent results from different states (e.g. wallet balance may be as of a different block from the chain state reported)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-netinfo", "Get network peer connection information from the remote server. An optional integer argument from 0 to 4 can be passed for different peers listings (default: 0). Pass \"help\" for detailed help documentation.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 3e8e5fc7bc..b84d909b07 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -33,51 +33,52 @@ static void SetupWalletToolArgs(ArgsManager& argsman) argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("info", "Get wallet info", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("create", "Create new wallet file", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("dump", "Print out all of the wallet key-value records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("createfromdump", "Create new wallet file from dumped records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); + argsman.AddCommand("info", "Get wallet info", OptionsCategory::COMMANDS); + argsman.AddCommand("create", "Create new wallet file", OptionsCategory::COMMANDS); + argsman.AddCommand("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", OptionsCategory::COMMANDS); + argsman.AddCommand("dump", "Print out all of the wallet key-value records", OptionsCategory::COMMANDS); + argsman.AddCommand("createfromdump", "Create new wallet file from dumped records", OptionsCategory::COMMANDS); } -static bool WalletAppInit(int argc, char* argv[]) +static bool WalletAppInit(ArgsManager& args, int argc, char* argv[]) { - SetupWalletToolArgs(gArgs); + SetupWalletToolArgs(args); std::string error_message; - if (!gArgs.ParseParameters(argc, argv, error_message)) { + if (!args.ParseParameters(argc, argv, error_message)) { tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error_message); return false; } - if (argc < 2 || HelpRequested(gArgs) || gArgs.IsArgSet("-version")) { + if (argc < 2 || HelpRequested(args) || args.IsArgSet("-version")) { std::string strUsage = strprintf("%s bitcoin-wallet version", PACKAGE_NAME) + " " + FormatFullVersion() + "\n"; - if (!gArgs.IsArgSet("-version")) { - strUsage += "\n" - "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" - "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" - "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" - "Usage:\n" - " bitcoin-wallet [options] <command>\n"; - strUsage += "\n" + gArgs.GetHelpMessage(); - } + if (!args.IsArgSet("-version")) { + strUsage += "\n" + "bitcoin-wallet is an offline tool for creating and interacting with " PACKAGE_NAME " wallet files.\n" + "By default bitcoin-wallet will act on wallets in the default mainnet wallet directory in the datadir.\n" + "To change the target wallet, use the -datadir, -wallet and -testnet/-regtest arguments.\n\n" + "Usage:\n" + " bitcoin-wallet [options] <command>\n"; + strUsage += "\n" + args.GetHelpMessage(); + } tfm::format(std::cout, "%s", strUsage); return false; } // check for printtoconsole, allow -debug - LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", gArgs.GetBoolArg("-debug", false)); + LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", args.GetBoolArg("-debug", false)); if (!CheckDataDirOption()) { - tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", gArgs.GetArg("-datadir", "")); + tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", args.GetArg("-datadir", "")); return false; } // Check for chain settings (Params() calls are only valid after this clause) - SelectParams(gArgs.GetChainName()); + SelectParams(args.GetChainName()); return true; } int main(int argc, char* argv[]) { + ArgsManager& args = gArgs; #ifdef WIN32 util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); @@ -85,7 +86,7 @@ int main(int argc, char* argv[]) SetupEnvironment(); RandomInit(); try { - if (!WalletAppInit(argc, argv)) return EXIT_FAILURE; + if (!WalletAppInit(args, argc, argv)) return EXIT_FAILURE; } catch (const std::exception& e) { PrintExceptionContinue(&e, "WalletAppInit()"); return EXIT_FAILURE; @@ -94,33 +95,19 @@ int main(int argc, char* argv[]) return EXIT_FAILURE; } - std::string method {}; - for(int i = 1; i < argc; ++i) { - if (!IsSwitchChar(argv[i][0])) { - if (!method.empty()) { - tfm::format(std::cerr, "Error: two methods provided (%s and %s). Only one method should be provided.\n", method, argv[i]); - return EXIT_FAILURE; - } - method = argv[i]; - } - } - - if (method.empty()) { + const auto command = args.GetCommand(); + if (!command) { tfm::format(std::cerr, "No method provided. Run `bitcoin-wallet -help` for valid methods.\n"); return EXIT_FAILURE; } - - // A name must be provided when creating a file - if (method == "create" && !gArgs.IsArgSet("-wallet")) { - tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); + if (command->args.size() != 0) { + tfm::format(std::cerr, "Error: Additional arguments provided (%s). Methods do not take arguments. Please refer to `-help`.\n", Join(command->args, ", ")); return EXIT_FAILURE; } - std::string name = gArgs.GetArg("-wallet", ""); - ECCVerifyHandle globalVerifyHandle; ECC_Start(); - if (!WalletTool::ExecuteWalletToolFunc(gArgs, method, name)) { + if (!WalletTool::ExecuteWalletToolFunc(args, command->command)) { return EXIT_FAILURE; } ECC_Stop(); diff --git a/src/checkqueue.h b/src/checkqueue.h index e3faa1dec0..4ceeb3600a 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -6,13 +6,12 @@ #define BITCOIN_CHECKQUEUE_H #include <sync.h> +#include <tinyformat.h> +#include <util/threadnames.h> #include <algorithm> #include <vector> -#include <boost/thread/condition_variable.hpp> -#include <boost/thread/mutex.hpp> - template <typename T> class CCheckQueueControl; @@ -31,61 +30,64 @@ class CCheckQueue { private: //! Mutex to protect the inner state - boost::mutex mutex; + Mutex m_mutex; //! Worker threads block on this when out of work - boost::condition_variable condWorker; + std::condition_variable m_worker_cv; //! Master thread blocks on this when out of work - boost::condition_variable condMaster; + std::condition_variable m_master_cv; //! The queue of elements to be processed. //! As the order of booleans doesn't matter, it is used as a LIFO (stack) - std::vector<T> queue; + std::vector<T> queue GUARDED_BY(m_mutex); //! The number of workers (including the master) that are idle. - int nIdle; + int nIdle GUARDED_BY(m_mutex){0}; //! The total number of workers (including the master). - int nTotal; + int nTotal GUARDED_BY(m_mutex){0}; //! The temporary evaluation result. - bool fAllOk; + bool fAllOk GUARDED_BY(m_mutex){true}; /** * Number of verifications that haven't completed yet. * This includes elements that are no longer queued, but still in the * worker's own batches. */ - unsigned int nTodo; + unsigned int nTodo GUARDED_BY(m_mutex){0}; //! The maximum number of elements to be processed in one batch - unsigned int nBatchSize; + const unsigned int nBatchSize; + + std::vector<std::thread> m_worker_threads; + bool m_request_stop GUARDED_BY(m_mutex){false}; /** Internal function that does bulk of the verification work. */ - bool Loop(bool fMaster = false) + bool Loop(bool fMaster) { - boost::condition_variable& cond = fMaster ? condMaster : condWorker; + std::condition_variable& cond = fMaster ? m_master_cv : m_worker_cv; std::vector<T> vChecks; vChecks.reserve(nBatchSize); unsigned int nNow = 0; bool fOk = true; do { { - boost::unique_lock<boost::mutex> lock(mutex); + WAIT_LOCK(m_mutex, lock); // first do the clean-up of the previous loop run (allowing us to do it in the same critsect) if (nNow) { fAllOk &= fOk; nTodo -= nNow; if (nTodo == 0 && !fMaster) // We processed the last element; inform the master it can exit and return the result - condMaster.notify_one(); + m_master_cv.notify_one(); } else { // first iteration nTotal++; } // logically, the do loop starts here - while (queue.empty()) { + while (queue.empty() && !m_request_stop) { if (fMaster && nTodo == 0) { nTotal--; bool fRet = fAllOk; @@ -98,6 +100,10 @@ private: cond.wait(lock); // wait nIdle--; } + if (m_request_stop) { + return false; + } + // Decide how many work units to process now. // * Do not try to do everything at once, but aim for increasingly smaller batches so // all workers finish approximately simultaneously. @@ -106,7 +112,7 @@ private: nNow = std::max(1U, std::min(nBatchSize, (unsigned int)queue.size() / (nTotal + nIdle + 1))); vChecks.resize(nNow); for (unsigned int i = 0; i < nNow; i++) { - // We want the lock on the mutex to be as short as possible, so swap jobs from the global + // We want the lock on the m_mutex to be as short as possible, so swap jobs from the global // queue to the local batch vector instead of copying. vChecks[i].swap(queue.back()); queue.pop_back(); @@ -124,40 +130,68 @@ private: public: //! Mutex to ensure only one concurrent CCheckQueueControl - boost::mutex ControlMutex; + Mutex m_control_mutex; //! Create a new check queue - explicit CCheckQueue(unsigned int nBatchSizeIn) : nIdle(0), nTotal(0), fAllOk(true), nTodo(0), nBatchSize(nBatchSizeIn) {} + explicit CCheckQueue(unsigned int nBatchSizeIn) + : nBatchSize(nBatchSizeIn) + { + } - //! Worker thread - void Thread() + //! Create a pool of new worker threads. + void StartWorkerThreads(const int threads_num) { - Loop(); + { + LOCK(m_mutex); + nIdle = 0; + nTotal = 0; + fAllOk = true; + } + assert(m_worker_threads.empty()); + for (int n = 0; n < threads_num; ++n) { + m_worker_threads.emplace_back([this, n]() { + util::ThreadRename(strprintf("scriptch.%i", n)); + Loop(false /* worker thread */); + }); + } } //! Wait until execution finishes, and return whether all evaluations were successful. bool Wait() { - return Loop(true); + return Loop(true /* master thread */); } //! Add a batch of checks to the queue void Add(std::vector<T>& vChecks) { - boost::unique_lock<boost::mutex> lock(mutex); + LOCK(m_mutex); for (T& check : vChecks) { queue.push_back(T()); check.swap(queue.back()); } nTodo += vChecks.size(); if (vChecks.size() == 1) - condWorker.notify_one(); + m_worker_cv.notify_one(); else if (vChecks.size() > 1) - condWorker.notify_all(); + m_worker_cv.notify_all(); + } + + //! Stop all of the worker threads. + void StopWorkerThreads() + { + WITH_LOCK(m_mutex, m_request_stop = true); + m_worker_cv.notify_all(); + for (std::thread& t : m_worker_threads) { + t.join(); + } + m_worker_threads.clear(); + WITH_LOCK(m_mutex, m_request_stop = false); } ~CCheckQueue() { + assert(m_worker_threads.empty()); } }; @@ -181,7 +215,7 @@ public: { // passed queue is supposed to be unused, or nullptr if (pqueue != nullptr) { - ENTER_CRITICAL_SECTION(pqueue->ControlMutex); + ENTER_CRITICAL_SECTION(pqueue->m_control_mutex); } } @@ -205,7 +239,7 @@ public: if (!fDone) Wait(); if (pqueue != nullptr) { - LEAVE_CRITICAL_SECTION(pqueue->ControlMutex); + LEAVE_CRITICAL_SECTION(pqueue->m_control_mutex); } } }; diff --git a/src/clientversion.h b/src/clientversion.h index 2da909f829..0ed3f68094 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CLIENTVERSION_H #define BITCOIN_CLIENTVERSION_H +#include <util/macros.h> + #if defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> #endif //HAVE_CONFIG_H @@ -14,13 +16,6 @@ #error Client version information missing: version is not defined by bitcoin-config.h or in any other way #endif -/** - * Converts the parameter X to a string after macro replacement on X has been performed. - * Don't merge these into one macro! - */ -#define STRINGIZE(X) DO_STRINGIZE(X) -#define DO_STRINGIZE(X) #X - //! Copyright string used in Windows .rc files #define COPYRIGHT_STR "2009-" STRINGIZE(COPYRIGHT_YEAR) " " COPYRIGHT_HOLDERS_FINAL diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp index 6d7a293f9b..8a51f310f7 100644 --- a/src/compat/glibc_compat.cpp +++ b/src/compat/glibc_compat.cpp @@ -54,6 +54,12 @@ __asm(".symver log2f_old,log2f@GLIBC_2.2.5"); __asm(".symver log2f_old,log2f@GLIBC_2.4"); #elif defined(__aarch64__) __asm(".symver log2f_old,log2f@GLIBC_2.17"); +#elif defined(__powerpc64__) +# ifdef WORDS_BIGENDIAN +__asm(".symver log2f_old,log2f@GLIBC_2.3"); +# else +__asm(".symver log2f_old,log2f@GLIBC_2.17"); +# endif #elif defined(__riscv) __asm(".symver log2f_old,log2f@GLIBC_2.27"); #endif diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 33e4b366a1..c119036db2 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -8,9 +8,10 @@ #include <clientversion.h> #include <fs.h> #include <serialize.h> +#include <span.h> #include <streams.h> -#include <util/system.h> #include <util/strencodings.h> +#include <util/system.h> #include <leveldb/db.h> #include <leveldb/write_batch.h> @@ -73,12 +74,12 @@ public: { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE); ssValue << value; ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); - leveldb::Slice slValue(ssValue.data(), ssValue.size()); + leveldb::Slice slValue((const char*)ssValue.data(), ssValue.size()); batch.Put(slKey, slValue); // LevelDB serializes writes as: @@ -98,7 +99,7 @@ public: { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); batch.Delete(slKey); // LevelDB serializes erases as: @@ -137,7 +138,7 @@ public: CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); piter->Seek(slKey); } @@ -146,7 +147,7 @@ public: template<typename K> bool GetKey(K& key) { leveldb::Slice slKey = piter->key(); try { - CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); + CDataStream ssKey(MakeUCharSpan(slKey), SER_DISK, CLIENT_VERSION); ssKey >> key; } catch (const std::exception&) { return false; @@ -157,7 +158,7 @@ public: template<typename V> bool GetValue(V& value) { leveldb::Slice slValue = piter->value(); try { - CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); + CDataStream ssValue(MakeUCharSpan(slValue), SER_DISK, CLIENT_VERSION); ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); ssValue >> value; } catch (const std::exception&) { @@ -232,7 +233,7 @@ public: CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); @@ -243,7 +244,7 @@ public: dbwrapper_private::HandleError(status); } try { - CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); + CDataStream ssValue(MakeUCharSpan(strValue), SER_DISK, CLIENT_VERSION); ssValue.Xor(obfuscate_key); ssValue >> value; } catch (const std::exception&) { @@ -266,7 +267,7 @@ public: CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); @@ -310,8 +311,8 @@ public: ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; ssKey2 << key_end; - leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); - leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + leveldb::Slice slKey1((const char*)ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2((const char*)ssKey2.data(), ssKey2.size()); uint64_t size = 0; leveldb::Range range(slKey1, slKey2); pdb->GetApproximateSizes(&range, 1, &size); @@ -329,11 +330,10 @@ public: ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; ssKey2 << key_end; - leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); - leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + leveldb::Slice slKey1((const char*)ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2((const char*)ssKey2.data(), ssKey2.size()); pdb->CompactRange(&slKey1, &slKey2); } - }; #endif // BITCOIN_DBWRAPPER_H diff --git a/src/index/base.cpp b/src/index/base.cpp index e67b813763..3d3dda95b1 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -62,7 +62,7 @@ bool BaseIndex::Init() if (locator.IsNull()) { m_best_block_index = nullptr; } else { - m_best_block_index = FindForkInGlobalIndex(::ChainActive(), locator); + m_best_block_index = g_chainman.m_blockman.FindForkInGlobalIndex(::ChainActive(), locator); } m_synced = m_best_block_index.load() == ::ChainActive().Tip(); return true; @@ -239,7 +239,7 @@ void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) const CBlockIndex* locator_tip_index; { LOCK(cs_main); - locator_tip_index = LookupBlockIndex(locator_tip_hash); + locator_tip_index = g_chainman.m_blockman.LookupBlockIndex(locator_tip_hash); } if (!locator_tip_index) { diff --git a/src/init.cpp b/src/init.cpp index 09eb76eaee..a77e2cf1cb 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -68,6 +68,8 @@ #include <set> #include <stdint.h> #include <stdio.h> +#include <thread> +#include <vector> #ifndef WIN32 #include <attributes.h> @@ -78,7 +80,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/signals2/signal.hpp> -#include <boost/thread/thread.hpp> #if ENABLE_ZMQ #include <zmq/zmqabstractnotifier.h> @@ -155,8 +156,6 @@ static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; static std::thread g_load_block; -static boost::thread_group threadGroup; - void Interrupt(NodeContext& node) { InterruptHTTPServer(); @@ -218,11 +217,10 @@ void Shutdown(NodeContext& node) StopTorControl(); // After everything has been shut down, but before things get flushed, stop the - // CScheduler/checkqueue, threadGroup and load block thread. + // CScheduler/checkqueue, scheduler and load block thread. if (node.scheduler) node.scheduler->stop(); if (g_load_block.joinable()) g_load_block.join(); - threadGroup.interrupt_all(); - threadGroup.join_all(); + StopScriptCheckWorkerThreads(); // After the threads that potentially access these pointers have been stopped, // destruct and reset all to nullptr. @@ -447,7 +445,7 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'download' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-onlynet=<net>", "Make outgoing connections only through network <net> (ipv4, ipv6 or onion). Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple networks. Warning: if it is used with ipv4 or ipv6 but not onion and the -onion or -proxy option is set, then outbound onion connections will still be made; use -noonion or -onion=0 to disable outbound onion connections in this case.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -456,8 +454,8 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); - argsman.AddArg("-timeout=<n>", strprintf("Specify connection timeout in milliseconds (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-peertimeout=<n>", strprintf("Specify p2p connection timeout in seconds. This option determines the amount of time a peer may be inactive before the connection to it is dropped. (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-timeout=<n>", strprintf("Specify socket connection timeout in milliseconds. If an initial attempt to connect is unsuccessful after this amount of time, drop it (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-peertimeout=<n>", strprintf("Specify a p2p connection timeout delay in seconds. After connecting to a peer, wait this amount of time before considering disconnection based on inactivity (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-torcontrol=<ip>:<port>", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-torpassword=<pass>", "Tor control port password (default: empty)", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::CONNECTION); #ifdef USE_UPNP @@ -521,10 +519,11 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_BOOL | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " - "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ".", + "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ". This option can be specified multiple times to output multiple categories.", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except the specified category. This option can be specified multiple times to exclude multiple categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); #ifdef HAVE_THREAD_LOCAL @@ -704,7 +703,7 @@ static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImp if (!file) break; // This error is logged in OpenBlockFile LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); - LoadExternalBlockFile(chainparams, file, &pos); + ::ChainstateActive().LoadExternalBlockFile(chainparams, file, &pos); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; @@ -723,7 +722,7 @@ static void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImp FILE *file = fsbridge::fopen(path, "rb"); if (file) { LogPrintf("Importing blocks file %s...\n", path.string()); - LoadExternalBlockFile(chainparams, file); + ::ChainstateActive().LoadExternalBlockFile(chainparams, file); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; @@ -1042,16 +1041,17 @@ bool AppInitParameterInteraction(const ArgsManager& args) // Trim requested connection counts, to fit into system limitations // <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695 - nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind); + nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind + NUM_FDS_MESSAGE_CAPTURE); + #ifdef USE_POLL int fd_max = nFD; #else int fd_max = FD_SETSIZE; #endif - nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS), 0); + nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS - NUM_FDS_MESSAGE_CAPTURE), 0); if (nFD < MIN_CORE_FILEDESCRIPTORS) return InitError(_("Not enough file descriptors available.")); - nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS, nMaxConnections); + nMaxConnections = std::min(nFD - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS - NUM_FDS_MESSAGE_CAPTURE, nMaxConnections); if (nMaxConnections < nUserMaxConnections) InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); @@ -1334,16 +1334,14 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA LogPrintf("Script verification uses %d additional threads\n", script_threads); if (script_threads >= 1) { g_parallel_script_checks = true; - for (int i = 0; i < script_threads; ++i) { - threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); - } + StartScriptCheckWorkerThreads(script_threads); } assert(!node.scheduler); node.scheduler = MakeUnique<CScheduler>(); // Start the lightweight task scheduler thread - threadGroup.create_thread([&] { TraceThread("scheduler", [&] { node.scheduler->serviceQueue(); }); }); + node.scheduler->m_service_thread = std::thread([&] { TraceThread("scheduler", [&] { node.scheduler->serviceQueue(); }); }); // Gather some entropy once per minute. node.scheduler->scheduleEvery([]{ @@ -1612,7 +1610,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node, interfaces::BlockA // If the loaded chain has a wrong genesis, bail out immediately // (we're likely using a testnet datadir, or the other way around). if (!chainman.BlockIndex().empty() && - !LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { + !g_chainman.m_blockman.LookupBlockIndex(chainparams.GetConsensus().hashGenesisBlock)) { return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); } diff --git a/src/key_io.cpp b/src/key_io.cpp index a270ede864..e27673fd16 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -12,6 +12,9 @@ #include <assert.h> #include <string.h> +/// Maximum witness length for Bech32 addresses. +static constexpr std::size_t BECH32_WITNESS_PROG_MAX_LEN = 40; + namespace { class DestinationEncoder { @@ -65,10 +68,11 @@ public: std::string operator()(const CNoDestination& no) const { return {}; } }; -CTxDestination DecodeDestination(const std::string& str, const CChainParams& params) +CTxDestination DecodeDestination(const std::string& str, const CChainParams& params, std::string& error_str) { std::vector<unsigned char> data; uint160 hash; + error_str = ""; if (DecodeBase58Check(str, data, 21)) { // base58-encoded Bitcoin addresses. // Public-key-hash-addresses have version 0 (or 111 testnet). @@ -85,10 +89,21 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par std::copy(data.begin() + script_prefix.size(), data.end(), hash.begin()); return ScriptHash(hash); } + + // Set potential error message. + // This message may be changed if the address can also be interpreted as a Bech32 address. + error_str = "Invalid prefix for Base58-encoded address"; } data.clear(); auto bech = bech32::Decode(str); - if (bech.second.size() > 0 && bech.first == params.Bech32HRP()) { + if (bech.second.size() > 0) { + error_str = ""; + + if (bech.first != params.Bech32HRP()) { + error_str = "Invalid prefix for Bech32 address"; + return CNoDestination(); + } + // Bech32 decoding int version = bech.second[0]; // The first 5 bit symbol is the witness version (0-16) // The rest of the symbols are converted witness program bytes. @@ -109,11 +124,21 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par return scriptid; } } + + error_str = "Invalid Bech32 v0 address data size"; + return CNoDestination(); + } + + if (version > 16) { + error_str = "Invalid Bech32 address witness version"; return CNoDestination(); } - if (version > 16 || data.size() < 2 || data.size() > 40) { + + if (data.size() < 2 || data.size() > BECH32_WITNESS_PROG_MAX_LEN) { + error_str = "Invalid Bech32 address data size"; return CNoDestination(); } + WitnessUnknown unk; unk.version = version; std::copy(data.begin(), data.end(), unk.program); @@ -121,6 +146,10 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par return unk; } } + + // Set error message if address can't be interpreted as Base58 or Bech32. + if (error_str.empty()) error_str = "Invalid address format"; + return CNoDestination(); } } // namespace @@ -208,14 +237,21 @@ std::string EncodeDestination(const CTxDestination& dest) return std::visit(DestinationEncoder(Params()), dest); } +CTxDestination DecodeDestination(const std::string& str, std::string& error_msg) +{ + return DecodeDestination(str, Params(), error_msg); +} + CTxDestination DecodeDestination(const std::string& str) { - return DecodeDestination(str, Params()); + std::string error_msg; + return DecodeDestination(str, error_msg); } bool IsValidDestinationString(const std::string& str, const CChainParams& params) { - return IsValidDestination(DecodeDestination(str, params)); + std::string error_msg; + return IsValidDestination(DecodeDestination(str, params, error_msg)); } bool IsValidDestinationString(const std::string& str) diff --git a/src/key_io.h b/src/key_io.h index d80c08f49c..bd81f7847e 100644 --- a/src/key_io.h +++ b/src/key_io.h @@ -23,6 +23,7 @@ std::string EncodeExtPubKey(const CExtPubKey& extpubkey); std::string EncodeDestination(const CTxDestination& dest); CTxDestination DecodeDestination(const std::string& str); +CTxDestination DecodeDestination(const std::string& str, std::string& error_msg); bool IsValidDestinationString(const std::string& str); bool IsValidDestinationString(const std::string& str, const CChainParams& params); diff --git a/src/miner.cpp b/src/miner.cpp index 009ae6b13a..076d43c951 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -45,7 +45,7 @@ void RegenerateCommitments(CBlock& block) tx.vout.erase(tx.vout.begin() + GetWitnessCommitmentIndex(block)); block.vtx.at(0) = MakeTransactionRef(tx); - GenerateCoinbaseCommitment(block, WITH_LOCK(cs_main, return LookupBlockIndex(block.hashPrevBlock)), Params().GetConsensus()); + GenerateCoinbaseCommitment(block, WITH_LOCK(cs_main, return g_chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock)), Params().GetConsensus()); block.hashMerkleRoot = BlockMerkleRoot(block); } @@ -176,7 +176,7 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblocktemplate->vTxSigOpsCost[0] = WITNESS_SCALE_FACTOR * GetLegacySigOpCount(*pblock->vtx[0]); BlockValidationState state; - if (!TestBlockValidity(state, chainparams, *pblock, pindexPrev, false, false)) { + if (!TestBlockValidity(state, chainparams, ::ChainstateActive(), *pblock, pindexPrev, false, false)) { throw std::runtime_error(strprintf("%s: TestBlockValidity failed: %s", __func__, state.ToString())); } int64_t nTime2 = GetTimeMicros(); diff --git a/src/net.cpp b/src/net.cpp index bd837eaa1a..d004aace88 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -35,6 +35,7 @@ #include <algorithm> #include <cstdint> +#include <functional> #include <unordered_map> #include <math.h> @@ -814,7 +815,7 @@ size_t CConnman::SocketSendData(CNode& node) const // error int nErr = WSAGetLastError(); if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) { - LogPrintf("socket send error %s\n", NetworkErrorString(nErr)); + LogPrint(BCLog::NET, "socket send error for peer=%d: %s\n", node.GetId(), NetworkErrorString(nErr)); node.CloseSocketDisconnect(); } } @@ -1003,6 +1004,7 @@ bool CConnman::AttemptToEvictConnection() LOCK(cs_vNodes); for (CNode* pnode : vNodes) { if (pnode->GetId() == *node_id_to_evict) { + LogPrint(BCLog::NET, "selected %s connection for eviction peer=%d; disconnecting\n", pnode->ConnectionTypeAsString(), pnode->GetId()); pnode->fDisconnect = true; return true; } @@ -1051,7 +1053,7 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { } if (!fNetworkActive) { - LogPrintf("connection from %s dropped: not accepting new connections\n", addr.ToString()); + LogPrint(BCLog::NET, "connection from %s dropped: not accepting new connections\n", addr.ToString()); CloseSocket(hSocket); return; } @@ -1229,17 +1231,17 @@ bool CConnman::InactivityCheck(const CNode& node) const } if (node.nLastRecv == 0 || node.nLastSend == 0) { - LogPrint(BCLog::NET, "socket no message in first %i seconds, %d %d from %d\n", m_peer_connect_timeout, node.nLastRecv != 0, node.nLastSend != 0, node.GetId()); + LogPrint(BCLog::NET, "socket no message in first %i seconds, %d %d peer=%d\n", m_peer_connect_timeout, node.nLastRecv != 0, node.nLastSend != 0, node.GetId()); return true; } if (now > node.nLastSend + TIMEOUT_INTERVAL) { - LogPrintf("socket sending timeout: %is\n", now - node.nLastSend); + LogPrint(BCLog::NET, "socket sending timeout: %is peer=%d\n", now - node.nLastSend, node.GetId()); return true; } if (now > node.nLastRecv + TIMEOUT_INTERVAL) { - LogPrintf("socket receive timeout: %is\n", now - node.nLastRecv); + LogPrint(BCLog::NET, "socket receive timeout: %is peer=%d\n", now - node.nLastRecv, node.GetId()); return true; } @@ -1247,12 +1249,12 @@ bool CConnman::InactivityCheck(const CNode& node) const // We use mockable time for ping timeouts. This means that setmocktime // may cause pings to time out for peers that have been connected for // longer than m_peer_connect_timeout. - LogPrintf("ping timeout: %fs\n", 0.000001 * count_microseconds(GetTime<std::chrono::microseconds>() - node.m_ping_start.load())); + LogPrint(BCLog::NET, "ping timeout: %fs peer=%d\n", 0.000001 * count_microseconds(GetTime<std::chrono::microseconds>() - node.m_ping_start.load()), node.GetId()); return true; } if (!node.fSuccessfullyConnected) { - LogPrint(BCLog::NET, "version handshake timeout from %d\n", node.GetId()); + LogPrint(BCLog::NET, "version handshake timeout peer=%d\n", node.GetId()); return true; } @@ -2692,6 +2694,7 @@ bool CConnman::DisconnectNode(const std::string& strNode) { LOCK(cs_vNodes); if (CNode* pnode = FindNode(strNode)) { + LogPrint(BCLog::NET, "disconnect by address%s matched peer=%d; disconnecting\n", (fLogIPs ? strprintf("=%s", strNode) : ""), pnode->GetId()); pnode->fDisconnect = true; return true; } @@ -2704,6 +2707,7 @@ bool CConnman::DisconnectNode(const CSubNet& subnet) LOCK(cs_vNodes); for (CNode* pnode : vNodes) { if (subnet.Match(pnode->addr)) { + LogPrint(BCLog::NET, "disconnect by subnet%s matched peer=%d; disconnecting\n", (fLogIPs ? strprintf("=%s", subnet.ToString()) : ""), pnode->GetId()); pnode->fDisconnect = true; disconnected = true; } @@ -2721,6 +2725,7 @@ bool CConnman::DisconnectNode(NodeId id) LOCK(cs_vNodes); for(CNode* pnode : vNodes) { if (id == pnode->GetId()) { + LogPrint(BCLog::NET, "disconnect by id peer=%d; disconnecting\n", pnode->GetId()); pnode->fDisconnect = true; return true; } @@ -2874,6 +2879,9 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.m_type), nMessageSize, pnode->GetId()); + if (gArgs.GetBoolArg("-capturemessages", false)) { + CaptureMessage(pnode->addr, msg.m_type, msg.data, /* incoming */ false); + } // make sure we use the appropriate network transport format std::vector<unsigned char> serializedHeader; @@ -2889,18 +2897,14 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) pnode->mapSendBytesPerMsgCmd[msg.m_type] += nTotalSize; pnode->nSendSize += nTotalSize; - if (pnode->nSendSize > nSendBufferMaxSize) - pnode->fPauseSend = true; + if (pnode->nSendSize > nSendBufferMaxSize) pnode->fPauseSend = true; pnode->vSendMsg.push_back(std::move(serializedHeader)); - if (nMessageSize) - pnode->vSendMsg.push_back(std::move(msg.data)); + if (nMessageSize) pnode->vSendMsg.push_back(std::move(msg.data)); // If write queue empty, attempt "optimistic write" - if (optimisticSend == true) - nBytesSent = SocketSendData(*pnode); + if (optimisticSend) nBytesSent = SocketSendData(*pnode); } - if (nBytesSent) - RecordBytesSent(nBytesSent); + if (nBytesSent) RecordBytesSent(nBytesSent); } bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) @@ -2943,3 +2947,31 @@ uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } + +void CaptureMessage(const CAddress& addr, const std::string& msg_type, const Span<const unsigned char>& data, bool is_incoming) +{ + // Note: This function captures the message at the time of processing, + // not at socket receive/send time. + // This ensures that the messages are always in order from an application + // layer (processing) perspective. + auto now = GetTime<std::chrono::microseconds>(); + + // Windows folder names can not include a colon + std::string clean_addr = addr.ToString(); + std::replace(clean_addr.begin(), clean_addr.end(), ':', '_'); + + fs::path base_path = GetDataDir() / "message_capture" / clean_addr; + fs::create_directories(base_path); + + fs::path path = base_path / (is_incoming ? "msgs_recv.dat" : "msgs_sent.dat"); + CAutoFile f(fsbridge::fopen(path, "ab"), SER_DISK, CLIENT_VERSION); + + ser_writedata64(f, now.count()); + f.write(msg_type.data(), msg_type.length()); + for (auto i = msg_type.length(); i < CMessageHeader::COMMAND_SIZE; ++i) { + f << '\0'; + } + uint32_t size = data.size(); + ser_writedata32(f, size); + f.write((const char*)data.data(), data.size()); +} @@ -20,6 +20,7 @@ #include <policy/feerate.h> #include <protocol.h> #include <random.h> +#include <span.h> #include <streams.h> #include <sync.h> #include <threadinterrupt.h> @@ -75,6 +76,8 @@ static constexpr uint64_t DEFAULT_MAX_UPLOAD_TARGET = 0; static const bool DEFAULT_BLOCKSONLY = false; /** -peertimeout default */ static const int64_t DEFAULT_PEER_CONNECT_TIMEOUT = 60; +/** Number of file descriptors required for message capture **/ +static const int NUM_FDS_MESSAGE_CAPTURE = 1; static const bool DEFAULT_FORCEDNSSEED = false; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; @@ -111,7 +114,8 @@ struct CSerializedNetMsg * connection. Aside from INBOUND, all types are initiated by us. * * If adding or removing types, please update CONNECTION_TYPE_DOC in - * src/rpc/net.cpp. */ + * src/rpc/net.cpp and src/qt/rpcconsole.cpp, as well as the descriptions in + * src/qt/guiutil.cpp and src/bitcoin-cli.cpp::NetinfoRequestHandler. */ enum class ConnectionType { /** * Inbound connections are those initiated by a peer. This is the only @@ -122,7 +126,7 @@ enum class ConnectionType { /** * These are the default connections that we use to connect with the - * network. There is no restriction on what is relayed- by default we relay + * network. There is no restriction on what is relayed; by default we relay * blocks, addresses & transactions. We automatically attempt to open * MAX_OUTBOUND_FULL_RELAY_CONNECTIONS using addresses from our AddrMan. */ @@ -130,8 +134,8 @@ enum class ConnectionType { /** - * We open manual connections to addresses that users explicitly inputted - * via the addnode RPC, or the -connect command line argument. Even if a + * We open manual connections to addresses that users explicitly requested + * via the addnode RPC or the -addnode/-connect configuration options. Even if a * manual connection is misbehaving, we do not automatically disconnect or * add it to our discouragement filter. */ @@ -150,7 +154,7 @@ enum class ConnectionType { * although in our codebase feeler connections encompass test-before-evict as well. * We make these connections approximately every FEELER_INTERVAL: * first we resolve previously found collisions if they exist (test-before-evict), - * otherwise connect to a node from the new table. + * otherwise we connect to a node from the new table. */ FEELER, @@ -441,6 +445,7 @@ public: * messages, implying a preference to receive ADDRv2 instead of ADDR ones. */ std::atomic_bool m_wants_addrv2{false}; + /** fSuccessfullyConnected is set to true on receiving VERACK from the peer. */ std::atomic_bool fSuccessfullyConnected{false}; // Setting fDisconnect to true will cause the node to be disconnected the // next time DisconnectNodes() runs @@ -1240,6 +1245,9 @@ inline std::chrono::microseconds PoissonNextSend(std::chrono::microseconds now, return std::chrono::microseconds{PoissonNextSend(now.count(), average_interval.count())}; } +/** Dump binary message to file, with timestamp */ +void CaptureMessage(const CAddress& addr, const std::string& msg_type, const Span<const unsigned char>& data, bool is_incoming); + struct NodeEvictionCandidate { NodeId id; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index cf73b1dae2..a246c8fa1c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -676,7 +676,7 @@ static void ProcessBlockAvailability(NodeId nodeid) EXCLUSIVE_LOCKS_REQUIRED(cs_ assert(state != nullptr); if (!state->hashLastUnknownBlock.IsNull()) { - const CBlockIndex* pindex = LookupBlockIndex(state->hashLastUnknownBlock); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(state->hashLastUnknownBlock); if (pindex && pindex->nChainWork > 0) { if (state->pindexBestKnownBlock == nullptr || pindex->nChainWork >= state->pindexBestKnownBlock->nChainWork) { state->pindexBestKnownBlock = pindex; @@ -693,7 +693,7 @@ static void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) EXCLUSIV ProcessBlockAvailability(nodeid); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (pindex && pindex->nChainWork > 0) { // An actually better block was announced. if (state->pindexBestKnownBlock == nullptr || pindex->nChainWork >= state->pindexBestKnownBlock->nChainWork) { @@ -880,13 +880,14 @@ void PeerManagerImpl::PushNodeVersion(CNode& pnode, int64_t nTime) CAddress(CService(), addr.nServices); CAddress addrMe = CAddress(CService(), nLocalNodeServices); + const bool tx_relay = !m_ignore_incoming_txs && pnode.m_tx_relay != nullptr; m_connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, - nonce, strSubVersion, nNodeStartingHeight, !m_ignore_incoming_txs && pnode.m_tx_relay != nullptr)); + nonce, strSubVersion, nNodeStartingHeight, tx_relay)); if (fLogIPs) { - LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), nodeid); + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), tx_relay, nodeid); } else { - LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), nodeid); + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), tx_relay, nodeid); } } @@ -1595,7 +1596,7 @@ bool static AlreadyHaveTx(const GenTxid& gtxid, const CTxMemPool& mempool) EXCLU bool static AlreadyHaveBlock(const uint256& block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - return LookupBlockIndex(block_hash) != nullptr; + return g_chainman.m_blockman.LookupBlockIndex(block_hash) != nullptr; } void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& connman) @@ -1685,7 +1686,7 @@ void static ProcessGetBlockData(CNode& pfrom, Peer& peer, const CChainParams& ch bool need_activate_chain = false; { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(inv.hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(inv.hash); if (pindex) { if (pindex->HaveTxsDownloaded() && !pindex->IsValid(BLOCK_VALID_SCRIPTS) && pindex->IsValid(BLOCK_VALID_TREE)) { @@ -1700,13 +1701,13 @@ void static ProcessGetBlockData(CNode& pfrom, Peer& peer, const CChainParams& ch } // release cs_main before calling ActivateBestChain if (need_activate_chain) { BlockValidationState state; - if (!ActivateBestChain(state, chainparams, a_recent_block)) { + if (!::ChainstateActive().ActivateBestChain(state, chainparams, a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", state.ToString()); } } LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(inv.hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(inv.hash); if (pindex) { send = BlockRequestAllowed(pindex, consensusParams); if (!send) { @@ -1996,7 +1997,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, // don't connect before giving DoS points // - Once a headers message is received that is valid and does connect, // nUnconnectingHeaders gets reset back to 0. - if (!LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { + if (!g_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) && nCount < MAX_BLOCKS_TO_ANNOUNCE) { nodestate->nUnconnectingHeaders++; m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", @@ -2026,7 +2027,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, // If we don't have the last header, then they'll have given us // something new (if these headers are valid). - if (!LookupBlockIndex(hashLastBlock)) { + if (!g_chainman.m_blockman.LookupBlockIndex(hashLastBlock)) { received_new_header = true; } } @@ -2278,7 +2279,7 @@ static bool PrepareBlockFilterRequest(CNode& peer, const CChainParams& chain_par { LOCK(cs_main); - stop_index = LookupBlockIndex(stop_hash); + stop_index = g_chainman.m_blockman.LookupBlockIndex(stop_hash); // Check that the stop block exists and the peer would be allowed to fetch it. if (!stop_index || !BlockRequestAllowed(stop_index, chain_params.GetConsensus())) { @@ -2642,9 +2643,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (fLogIPs) remoteAddr = ", peeraddr=" + pfrom.addr.ToString(); - LogPrint(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, peer=%d%s\n", + LogPrint(BCLog::NET, "receive version message: %s: version %d, blocks=%d, us=%s, txrelay=%d, peer=%d%s\n", cleanSubVer, pfrom.nVersion, - peer->m_starting_height, addrMe.ToString(), pfrom.GetId(), + peer->m_starting_height, addrMe.ToString(), fRelay, pfrom.GetId(), remoteAddr); int64_t nTimeOffset = nTime - GetTime(); @@ -2659,6 +2660,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Feeler connections exist only to verify if address is online. if (pfrom.IsFeelerConn()) { + LogPrint(BCLog::NET, "feeler connection completed peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; } return; @@ -2674,7 +2676,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); if (msg_type == NetMsgType::VERACK) { - if (pfrom.fSuccessfullyConnected) return; + if (pfrom.fSuccessfullyConnected) { + LogPrint(BCLog::NET, "ignoring redundant verack message from peer=%d\n", pfrom.GetId()); + return; + } if (!pfrom.IsInboundConn()) { LogPrintf("New outbound peer connected: version: %d, blocks=%d, peer=%d%s (%s)\n", @@ -2740,12 +2745,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - // Feature negotiation of wtxidrelay must happen between VERSION and VERACK - // to avoid relay problems from switching after a connection is up. + // BIP339 defines feature negotiation of wtxidrelay, which must happen between + // VERSION and VERACK to avoid relay problems from switching after a connection is up. if (msg_type == NetMsgType::WTXIDRELAY) { if (pfrom.fSuccessfullyConnected) { - // Disconnect peers that send wtxidrelay message after VERACK; this - // must be negotiated between VERSION and VERACK. + // Disconnect peers that send a wtxidrelay message after VERACK. + LogPrint(BCLog::NET, "wtxidrelay received after verack from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -2754,15 +2759,21 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!State(pfrom.GetId())->m_wtxid_relay) { State(pfrom.GetId())->m_wtxid_relay = true; g_wtxid_relay_peers++; + } else { + LogPrint(BCLog::NET, "ignoring duplicate wtxidrelay from peer=%d\n", pfrom.GetId()); } + } else { + LogPrint(BCLog::NET, "ignoring wtxidrelay due to old common version=%d from peer=%d\n", pfrom.GetCommonVersion(), pfrom.GetId()); } return; } + // BIP155 defines feature negotiation of addrv2 and sendaddrv2, which must happen + // between VERSION and VERACK. if (msg_type == NetMsgType::SENDADDRV2) { if (pfrom.fSuccessfullyConnected) { - // Disconnect peers that send SENDADDRV2 message after VERACK; this - // must be negotiated between VERSION and VERACK. + // Disconnect peers that send a SENDADDRV2 message after VERACK. + LogPrint(BCLog::NET, "sendaddrv2 received after verack from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -2789,6 +2800,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, s >> vAddr; if (!pfrom.RelayAddrsWithConn()) { + LogPrint(BCLog::NET, "ignoring %s message from %s peer=%d\n", msg_type, pfrom.ConnectionTypeAsString(), pfrom.GetId()); return; } if (vAddr.size() > MAX_ADDR_TO_SEND) @@ -2832,8 +2844,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_connman.AddNewAddresses(vAddrOk, pfrom.addr, 2 * 60 * 60); if (vAddr.size() < 1000) pfrom.fGetAddr = false; - if (pfrom.IsAddrFetchConn()) + if (pfrom.IsAddrFetchConn()) { + LogPrint(BCLog::NET, "addrfetch connection completed peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; + } return; } @@ -2960,7 +2974,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, a_recent_block = most_recent_block; } BlockValidationState state; - if (!ActivateBestChain(state, m_chainparams, a_recent_block)) { + if (!::ChainstateActive().ActivateBestChain(state, m_chainparams, a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", state.ToString()); } } @@ -2968,7 +2982,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LOCK(cs_main); // Find the last block the caller has in the main chain - const CBlockIndex* pindex = FindForkInGlobalIndex(::ChainActive(), locator); + const CBlockIndex* pindex = g_chainman.m_blockman.FindForkInGlobalIndex(::ChainActive(), locator); // Send the rest of the chain if (pindex) @@ -3021,7 +3035,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(req.blockhash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(req.blockhash); if (!pindex || !(pindex->nStatus & BLOCK_HAVE_DATA)) { LogPrint(BCLog::NET, "Peer %d sent us a getblocktxn for a block we don't have\n", pfrom.GetId()); return; @@ -3075,7 +3089,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (locator.IsNull()) { // If locator is null, return the hashStop block - pindex = LookupBlockIndex(hashStop); + pindex = g_chainman.m_blockman.LookupBlockIndex(hashStop); if (!pindex) { return; } @@ -3088,7 +3102,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, else { // Find the last block the caller has in the main chain - pindex = FindForkInGlobalIndex(::ChainActive(), locator); + pindex = g_chainman.m_blockman.FindForkInGlobalIndex(::ChainActive(), locator); if (pindex) pindex = ::ChainActive().Next(pindex); } @@ -3352,14 +3366,14 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, { LOCK(cs_main); - if (!LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { + if (!g_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers if (!::ChainstateActive().IsInitialBlockDownload()) m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, ::ChainActive().GetLocator(pindexBestHeader), uint256())); return; } - if (!LookupBlockIndex(cmpctblock.header.GetHash())) { + if (!g_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.GetHash())) { received_new_header = true; } } @@ -3840,6 +3854,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERLOAD) { if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + LogPrint(BCLog::NET, "filterload received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -3862,6 +3877,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERADD) { if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + LogPrint(BCLog::NET, "filteradd received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -3889,6 +3905,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (msg_type == NetMsgType::FILTERCLEAR) { if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + LogPrint(BCLog::NET, "filterclear received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; } @@ -4028,14 +4045,12 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt } // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - return false; + if (pfrom->fPauseSend) return false; std::list<CNetMessage> msgs; { LOCK(pfrom->cs_vProcessMsg); - if (pfrom->vProcessMsg.empty()) - return false; + if (pfrom->vProcessMsg.empty()) return false; // Just take one message msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); pfrom->nProcessQueueSize -= msgs.front().m_raw_message_size; @@ -4044,6 +4059,10 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt } CNetMessage& msg(msgs.front()); + if (gArgs.GetBoolArg("-capturemessages", false)) { + CaptureMessage(pfrom->addr, msg.m_command, MakeUCharSpan(msg.m_recv), /* incoming */ true); + } + msg.SetVersion(pfrom->GetCommonVersion()); const std::string& msg_type = msg.m_command; @@ -4425,7 +4444,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // then send all headers past that one. If we come across any // headers that aren't on ::ChainActive(), give up. for (const uint256& hash : peer->m_blocks_for_headers_relay) { - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); assert(pindex); if (::ChainActive()[pindex->nHeight] != pindex) { // Bail out if we reorged away from this block @@ -4517,7 +4536,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // in the past. if (!peer->m_blocks_for_headers_relay.empty()) { const uint256& hashToAnnounce = peer->m_blocks_for_headers_relay.back(); - const CBlockIndex* pindex = LookupBlockIndex(hashToAnnounce); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hashToAnnounce); assert(pindex); // Warn if we're announcing a block that is not on the main chain. diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 02e50c4dbe..56113cb731 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -63,7 +63,7 @@ static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, const stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); - stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; + stats.nHeight = g_chainman.m_blockman.LookupBlockIndex(stats.hashBlock)->nHeight; } PrepareHash(hash_obj, stats); diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index e07eaa33d8..b7efd68cfd 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -447,7 +447,7 @@ public: { LOCK(cs_main); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - if (CBlockIndex* fork = FindForkInGlobalIndex(active, locator)) { + if (CBlockIndex* fork = g_chainman.m_blockman.FindForkInGlobalIndex(active, locator)) { return fork->nHeight; } return nullopt; @@ -456,7 +456,7 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - return FillBlock(LookupBlockIndex(hash), block, lock, active); + return FillBlock(g_chainman.m_blockman.LookupBlockIndex(hash), block, lock, active); } bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override { @@ -468,7 +468,7 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - if (const CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (const CBlockIndex* block = g_chainman.m_blockman.LookupBlockIndex(block_hash)) { if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { return FillBlock(ancestor, ancestor_out, lock, active); } @@ -479,8 +479,8 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - const CBlockIndex* block = LookupBlockIndex(block_hash); - const CBlockIndex* ancestor = LookupBlockIndex(ancestor_hash); + const CBlockIndex* block = g_chainman.m_blockman.LookupBlockIndex(block_hash); + const CBlockIndex* ancestor = g_chainman.m_blockman.LookupBlockIndex(ancestor_hash); if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; return FillBlock(ancestor, ancestor_out, lock, active); } @@ -488,8 +488,8 @@ public: { WAIT_LOCK(cs_main, lock); const CChain& active = Assert(m_node.chainman)->ActiveChain(); - const CBlockIndex* block1 = LookupBlockIndex(block_hash1); - const CBlockIndex* block2 = LookupBlockIndex(block_hash2); + const CBlockIndex* block1 = g_chainman.m_blockman.LookupBlockIndex(block_hash1); + const CBlockIndex* block2 = g_chainman.m_blockman.LookupBlockIndex(block_hash2); const CBlockIndex* ancestor = block1 && block2 ? LastCommonAncestor(block1, block2) : nullptr; // Using & instead of && below to avoid short circuiting and leaving // output uninitialized. @@ -499,7 +499,7 @@ public: double guessVerificationProgress(const uint256& block_hash) override { LOCK(cs_main); - return GuessVerificationProgress(Params().TxData(), LookupBlockIndex(block_hash)); + return GuessVerificationProgress(Params().TxData(), g_chainman.m_blockman.LookupBlockIndex(block_hash)); } bool hasBlocks(const uint256& block_hash, int min_height, Optional<int> max_height) override { @@ -511,7 +511,7 @@ public: // used to limit the range, and passing min_height that's too low or // max_height that's too high will not crash or change the result. LOCK(::cs_main); - if (CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (CBlockIndex* block = g_chainman.m_blockman.LookupBlockIndex(block_hash)) { if (max_height && block->nHeight >= *max_height) block = block->GetAncestor(*max_height); for (; block->nStatus & BLOCK_HAVE_DATA; block = block->pprev) { // Check pprev to not segfault if min_height is too low diff --git a/src/psbt.cpp b/src/psbt.cpp index 4db57d3cd0..a849b2ea53 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -363,7 +363,7 @@ bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base6 bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) { - CDataStream ss_data(tx_data.data(), tx_data.data() + tx_data.size(), SER_NETWORK, PROTOCOL_VERSION); + CDataStream ss_data(MakeUCharSpan(tx_data), SER_NETWORK, PROTOCOL_VERSION); try { ss_data >> psbt; if (!ss_data.empty()) { diff --git a/src/qt/README.md b/src/qt/README.md index 30c68db15b..20c712c98d 100644 --- a/src/qt/README.md +++ b/src/qt/README.md @@ -1,10 +1,12 @@ -This directory contains the BitcoinQT graphical user interface (GUI). It uses the cross-platform framework [Qt](https://www1.qt.io/developers/). +This directory contains the source code for the Bitcoin Core graphical user interface (GUI). It uses the [Qt](https://www1.qt.io/developers/) cross-platform framework. The current precise version for Qt 5 is specified in [qt.mk](/depends/packages/qt.mk). ## Compile and run -See build instructions ([macOS](/doc/build-osx.md), [Windows](/doc/build-windows.md), [Unix](/doc/build-unix.md), etc). +See build instructions: [Unix](/doc/build-unix.md), [macOS](/doc/build-osx.md), [Windows](/doc/build-windows.md), [FreeBSD](/doc/build-freebsd.md), [NetBSD](/doc/build-netbsd.md), [OpenBSD](/doc/build-openbsd.md) + +When following your systems build instructions, make sure to install the `Qt` dependencies. To run: @@ -12,84 +14,111 @@ To run: ./src/qt/bitcoin-qt ``` -## Files and directories - -### forms +## Files and Directories -Contains [Designer UI](https://doc.qt.io/qt-5.9/designer-using-a-ui-file.html) files. They are created with [Qt Creator](#using-qt-creator-as-ide), but can be edited using any text editor. +#### forms/ -### locale +- A directory that contains [Designer UI](https://doc.qt.io/qt-5.9/designer-using-a-ui-file.html) files. These files specify the characteristics of form elements in XML. Qt UI files can be edited with [Qt Creator](#using-qt-creator-as-ide) or using any text editor. -Contains translations. They are periodically updated. The process is described [here](/doc/translation_process.md). +#### locale/ -### res +- Contains translations. They are periodically updated and an effort is made to support as many languages as possible. The process of contributing translations is described in [doc/translation_process.md](/doc/translation_process.md). -Resources such as the icon. +#### res/ -### test + - Contains graphical resources used to enhance the UI experience. -Tests. +#### test/ -### bitcoingui.(h/cpp) +- Functional tests used to ensure proper functionality of the GUI. Significant changes to the GUI code normally require new or updated tests. -Represents the main window of the Bitcoin UI. +#### bitcoingui.(h/cpp) -### \*model.(h/cpp) +- Represents the main window of the Bitcoin UI. -The model. When it has a corresponding controller, it generally inherits from [QAbstractTableModel](https://doc.qt.io/qt-5/qabstracttablemodel.html). Models that are used by controllers as helpers inherit from other Qt classes like [QValidator](https://doc.qt.io/qt-5/qvalidator.html). +#### \*model.(h/cpp) -ClientModel is used by the main application `bitcoingui` and several models like `peertablemodel`. +- The model. When it has a corresponding controller, it generally inherits from [QAbstractTableModel](https://doc.qt.io/qt-5/qabstracttablemodel.html). Models that are used by controllers as helpers inherit from other Qt classes like [QValidator](https://doc.qt.io/qt-5/qvalidator.html). +- ClientModel is used by the main application `bitcoingui` and several models like `peertablemodel`. -### \*page.(h/cpp) +#### \*page.(h/cpp) -A controller. `:NAMEpage.cpp` generally includes `:NAMEmodel.h` and `forms/:NAME.page.ui` with a similar `:NAME`. +- A controller. `:NAMEpage.cpp` generally includes `:NAMEmodel.h` and `forms/:NAME.page.ui` with a similar `:NAME`. -### \*dialog.(h/cpp) +#### \*dialog.(h/cpp) -Various dialogs, e.g. to open a URL. Inherit from [QDialog](https://doc.qt.io/qt-5/qdialog.html). +- Various dialogs, e.g. to open a URL. Inherit from [QDialog](https://doc.qt.io/qt-5/qdialog.html). -### paymentserver.(h/cpp) +#### paymentserver.(h/cpp) -Used to process BIP21 payment URI requests. Also handles URI based application switching (e.g. when following a bitcoin:... link from a browser). +- (Deprecated) Used to process BIP21 payment URI requests. Also handles URI-based application switching (e.g. when following a bitcoin:... link from a browser). -### walletview.(h/cpp) +#### walletview.(h/cpp) -Represents the view to a single wallet. +- Represents the view to a single wallet. -### Other .h/cpp files +#### Other .h/cpp files * UI elements like BitcoinAmountField, which inherit from QWidget. * `bitcoinstrings.cpp`: automatically generated -* `bitcoinunits.(h/cpp)`: BTC / mBTC / etc handling +* `bitcoinunits.(h/cpp)`: BTC / mBTC / etc. handling * `callback.h` -* `guiconstants.h`: UI colors, app name, etc +* `guiconstants.h`: UI colors, app name, etc. * `guiutil.h`: several helper functions * `macdockiconhandler.(h/mm)`: macOS dock icon handler * `macnotificationhandler.(h/mm)`: display notifications in macOS ## Contribute -See [CONTRIBUTING.md](/CONTRIBUTING.md) for general guidelines. Specifically for Qt: +See [CONTRIBUTING.md](/CONTRIBUTING.md) for general guidelines. + +**Note:** Do not change `local/bitcoin_en.ts`. It is updated [automatically](/doc/translation_process.md#writing-code-with-translations). + +## Using Qt Creator as an IDE + +[Qt Creator](https://www.qt.io/product/development-tools) is a powerful tool which packages a UI designer tool (Qt Designer) and a C++ IDE into one application. This is especially useful if you want to change the UI layout. + +#### Download Qt Creator + +On Unix and macOS, Qt Creator can be installed through your package manager. Alternatively, you can download a binary from the [Qt Website](https://www.qt.io/download/). + +**Note:** If installing from a binary grabbed from the Qt Website: During the installation process, uncheck everything except for `Qt Creator`. + +##### macOS + +```sh +brew install qt-creator +``` + +##### Ubuntu & Debian + +```sh +sudo apt-get install qtcreator +``` + +#### Setup Qt Creator -* don't change `local/bitcoin_en.ts`; this happens [automatically](/doc/translation_process.md#writing-code-with-translations) +1. Make sure you've installed all dependencies specified in your systems build instructions +2. Follow the compile instructions for your system, run `./configure` with the `--enable-debug` flag +3. Start Qt Creator. At the start page, do: `New` -> `Import Project` -> `Import Existing Project` +4. Enter `bitcoin-qt` as the Project Name and enter the absolute path to `src/qt` as Location +5. Check over the file selection, you may need to select the `forms` directory (necessary if you intend to edit *.ui files) +6. Confirm the `Summary` page +7. In the `Projects` tab, select `Manage Kits...` -## Using Qt Creator as IDE + **macOS** + - Under `Kits`: select the default "Desktop" kit + - Under `Compilers`: select `"Clang (x86 64bit in /usr/bin)"` + - Under `Debuggers`: select `"LLDB"` as debugger (you might need to set the path to your LLDB installation) -You can use Qt Creator as an IDE. This is especially useful if you want to change -the UI layout. + **Ubuntu & Debian** -Download and install the community edition of [Qt Creator](https://www.qt.io/download/). -Uncheck everything except Qt Creator during the installation process. + Note: Some of these options may already be set -Instructions for macOS: + - Under `Kits`: select the default "Desktop" kit + - Under `Compilers`: select `"GCC (x86 64bit in /usr/bin)"` + - Under `Debuggers`: select `"GDB"` as debugger -1. Make sure you installed everything through Homebrew mentioned in the [macOS build instructions](/doc/build-osx.md) -2. Use `./configure` with the `--enable-debug` flag -3. In Qt Creator do "New Project" -> Import Project -> Import Existing Project -4. Enter "bitcoin-qt" as project name, enter src/qt as location -5. Leave the file selection as it is -6. Confirm the "summary page" -7. In the "Projects" tab select "Manage Kits..." -8. Select the default "Desktop" kit and select "Clang (x86 64bit in /usr/bin)" as compiler -9. Select LLDB as debugger (you might need to set the path to your installation) -10. Start debugging with Qt Creator (you might need to the executable to "bitcoin-qt" under "Run", which is where you can also add command line arguments) +8. While in the `Projects` tab, ensure that you have the `bitcoin-qt` executable specified under `Run` + - If the executable is not specified: click `"Choose..."`, navigate to `src/qt`, and select `bitcoin-qt` +9. You're all set! Start developing, building, and debugging the Bitcoin Core GUI diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index 0fa52359a8..ab6168a541 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -56,7 +56,7 @@ protected: }; AddressBookPage::AddressBookPage(const PlatformStyle *platformStyle, Mode _mode, Tabs _tab, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::AddressBookPage), model(nullptr), mode(_mode), diff --git a/src/qt/askpassphrasedialog.cpp b/src/qt/askpassphrasedialog.cpp index c2349b78f9..5b1330b81b 100644 --- a/src/qt/askpassphrasedialog.cpp +++ b/src/qt/askpassphrasedialog.cpp @@ -20,7 +20,7 @@ #include <QPushButton> AskPassphraseDialog::AskPassphraseDialog(Mode _mode, QWidget *parent, SecureString* passphrase_out) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::AskPassphraseDialog), mode(_mode), model(nullptr), diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 8efb0e35d0..d6d5ba6968 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -51,6 +51,7 @@ #include <QThread> #include <QTimer> #include <QTranslator> +#include <QtGlobal> #if defined(QT_STATICPLUGIN) #include <QtPlugin> @@ -466,6 +467,13 @@ int GuiMain(int argc, char* argv[]) QCoreApplication::setAttribute(Qt::AA_EnableHighDpiScaling); #endif +#if (QT_VERSION <= QT_VERSION_CHECK(5, 9, 8)) && defined(Q_OS_MACOS) + const auto os_name = QSysInfo::prettyProductName(); + if (os_name.startsWith("macOS 11") || os_name.startsWith("macOS 10.16")) { + QApplication::setStyle("fusion"); + } +#endif + BitcoinApplication app; /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index f2a49e5a76..ccc0a6828c 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -30,6 +30,7 @@ #include <qt/macdockiconhandler.h> #endif +#include <functional> #include <chain.h> #include <chainparams.h> #include <interfaces/handler.h> @@ -845,7 +846,7 @@ void BitcoinGUI::showDebugWindowActivateConsole() void BitcoinGUI::showHelpMessageClicked() { - helpMessageDialog->show(); + GUIUtil::bringToFront(helpMessageDialog); } #ifdef ENABLE_WALLET diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index a2f46c339b..56813b2d19 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -19,6 +19,7 @@ #include <validation.h> #include <stdint.h> +#include <functional> #include <QDebug> #include <QThread> diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 08abef7866..ca78c96d70 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -42,7 +42,7 @@ bool CCoinControlWidgetItem::operator<(const QTreeWidgetItem &other) const { } CoinControlDialog::CoinControlDialog(CCoinControl& coin_control, WalletModel* _model, const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::CoinControlDialog), m_coin_control(coin_control), model(_model), diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index 70f080f4de..1467801522 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -9,10 +9,12 @@ #include <qt/createwalletdialog.h> #include <qt/forms/ui_createwalletdialog.h> +#include <qt/guiutil.h> + #include <QPushButton> CreateWalletDialog::CreateWalletDialog(QWidget* parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::CreateWalletDialog) { ui->setupUi(this); diff --git a/src/qt/editaddressdialog.cpp b/src/qt/editaddressdialog.cpp index cd25f856a2..e51ed9e656 100644 --- a/src/qt/editaddressdialog.cpp +++ b/src/qt/editaddressdialog.cpp @@ -13,7 +13,7 @@ EditAddressDialog::EditAddressDialog(Mode _mode, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::EditAddressDialog), mapper(nullptr), mode(_mode), diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index bce578a158..3831852185 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -1079,7 +1079,7 @@ <item row="1" column="0"> <widget class="QLabel" name="peerConnectionTypeLabel"> <property name="toolTip"> - <string>The type of peer connection:<ul><li>Inbound: initiated by peer</li><li>Outbound Full Relay: default</li><li>Outbound Block Relay: does not relay transactions or addresses</li><li>Outbound Manual: added using RPC %1 or %2/%3 configuration options</li><li>Outbound Feeler: short-lived, for testing addresses</li><li>Outbound Address Fetch: short-lived, for soliciting addresses</li></ul></string> + <string>The type of peer connection: %1</string> </property> <property name="text"> <string>Connection Type</string> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 430ecd322f..376acf0963 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -40,12 +40,14 @@ #include <QFontDatabase> #include <QFontMetrics> #include <QGuiApplication> +#include <QJsonObject> #include <QKeyEvent> #include <QLineEdit> #include <QList> #include <QLocale> #include <QMenu> #include <QMouseEvent> +#include <QPluginLoader> #include <QProgressDialog> #include <QScreen> #include <QSettings> @@ -764,10 +766,10 @@ QString NetworkToQString(Network net) assert(false); } -QString ConnectionTypeToQString(ConnectionType conn_type) +QString ConnectionTypeToQString(ConnectionType conn_type, bool relay_txes) { switch (conn_type) { - case ConnectionType::INBOUND: return QObject::tr("Inbound"); + case ConnectionType::INBOUND: return relay_txes ? QObject::tr("Inbound Full Relay") : QObject::tr("Inbound Block Relay"); case ConnectionType::OUTBOUND_FULL_RELAY: return QObject::tr("Outbound Full Relay"); case ConnectionType::BLOCK_RELAY: return QObject::tr("Outbound Block Relay"); case ConnectionType::MANUAL: return QObject::tr("Outbound Manual"); @@ -936,6 +938,20 @@ void LogQtInfo() const std::string plugin_link{"dynamic"}; #endif LogPrintf("Qt %s (%s), plugin=%s (%s)\n", qVersion(), qt_link, QGuiApplication::platformName().toStdString(), plugin_link); + const auto static_plugins = QPluginLoader::staticPlugins(); + if (static_plugins.empty()) { + LogPrintf("No static plugins.\n"); + } else { + LogPrintf("Static plugins:\n"); + for (const QStaticPlugin& p : static_plugins) { + QJsonObject meta_data = p.metaData(); + const std::string plugin_class = meta_data.take(QString("className")).toString().toStdString(); + const int plugin_version = meta_data.take(QString("version")).toInt(); + LogPrintf(" %s, version %d\n", plugin_class, plugin_version); + } + } + + LogPrintf("Style: %s / %s\n", QApplication::style()->objectName().toStdString(), QApplication::style()->metaObject()->className()); LogPrintf("System: %s, %s\n", QSysInfo::prettyProductName().toStdString(), QSysInfo::buildAbi().toStdString()); for (const QScreen* s : QGuiApplication::screens()) { LogPrintf("Screen: %s %dx%d, pixel ratio=%.1f\n", s->name().toStdString(), s->size().width(), s->size().height(), s->devicePixelRatio()); diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index c471b888f7..edfb5b13a2 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -45,6 +45,9 @@ QT_END_NAMESPACE */ namespace GUIUtil { + // Use this flags to prevent a "What's This" button in the title bar of the dialog on Windows. + constexpr auto dialog_flags = Qt::WindowTitleHint | Qt::WindowSystemMenuHint | Qt::WindowCloseButtonHint; + // Create human-readable string from date QString dateTimeStr(const QDateTime &datetime); QString dateTimeStr(qint64 nTime); @@ -230,7 +233,7 @@ namespace GUIUtil QString NetworkToQString(Network net); /** Convert enum ConnectionType to QString */ - QString ConnectionTypeToQString(ConnectionType conn_type); + QString ConnectionTypeToQString(ConnectionType conn_type, bool relay_txes); /** Convert seconds into a QString with days, hours, mins, secs */ QString formatDurationStr(int secs); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 235722d091..aa6b2665fa 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -119,7 +119,7 @@ int GetPruneTargetGB() } // namespace Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_size_gb) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::Intro), thread(nullptr), signalled(false), diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index 9a3d43c2a6..10bf82d532 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -11,7 +11,7 @@ #include <QUrl> OpenURIDialog::OpenURIDialog(QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::OpenURIDialog) { ui->setupUi(this); diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 416b9c83c9..7b8d7871ec 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -29,7 +29,7 @@ #include <QTimer> OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::OptionsDialog), model(nullptr), mapper(nullptr) diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 9f2e866272..bc542a0833 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -151,7 +151,6 @@ OverviewPage::OverviewPage(const PlatformStyle *platformStyle, QWidget *parent) // use a SingleColorIcon for the "out of sync warning" icon QIcon icon = platformStyle->SingleColorIcon(":/icons/warning"); - icon.addPixmap(icon.pixmap(QSize(64,64), QIcon::Normal), QIcon::Disabled); // also set the disabled icon because we are using a disabled QPushButton to work around missing HiDPI support of QLabel (https://bugreports.qt.io/browse/QTBUG-42503) ui->labelTransactionsStatus->setIcon(icon); ui->labelWalletStatus->setIcon(icon); diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 58167d4bb4..55ab6046cf 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -19,7 +19,7 @@ PSBTOperationsDialog::PSBTOperationsDialog( - QWidget* parent, WalletModel* wallet_model, ClientModel* client_model) : QDialog(parent), + QWidget* parent, WalletModel* wallet_model, ClientModel* client_model) : QDialog(parent, GUIUtil::dialog_flags), m_ui(new Ui::PSBTOperationsDialog), m_wallet_model(wallet_model), m_client_model(client_model) diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 1a9cd78d1e..0aea920c86 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -8,6 +8,7 @@ #include <qt/forms/ui_receivecoinsdialog.h> #include <qt/addresstablemodel.h> +#include <qt/guiutil.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> #include <qt/receiverequestdialog.h> @@ -21,7 +22,7 @@ #include <QTextDocument> ReceiveCoinsDialog::ReceiveCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::ReceiveCoinsDialog), columnResizingFixer(nullptr), model(nullptr), diff --git a/src/qt/receiverequestdialog.cpp b/src/qt/receiverequestdialog.cpp index bef3edc5fe..78ae5c07da 100644 --- a/src/qt/receiverequestdialog.cpp +++ b/src/qt/receiverequestdialog.cpp @@ -19,7 +19,7 @@ #endif ReceiveRequestDialog::ReceiveRequestDialog(QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::ReceiveRequestDialog), model(nullptr) { diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 18b913774b..03531a1381 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -181,7 +181,7 @@ void RecentRequestsTableModel::addNewRequest(const SendCoinsRecipient &recipient // called from ctor when loading from wallet void RecentRequestsTableModel::addNewRequest(const std::string &recipient) { - std::vector<char> data(recipient.begin(), recipient.end()); + std::vector<uint8_t> data(recipient.begin(), recipient.end()); CDataStream ss(data, SER_DISK, CLIENT_VERSION); RecentRequestEntry entry; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index df98dbbc99..a252685d2f 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -16,9 +16,10 @@ #include <chainparams.h> #include <interfaces/node.h> #include <netbase.h> -#include <rpc/server.h> #include <rpc/client.h> +#include <rpc/server.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/system.h> #include <util/threadnames.h> @@ -459,11 +460,22 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty ui->splitter->restoreState(settings.value("PeersTabSplitterSizes").toByteArray()); - QChar nonbreaking_hyphen(8209); + constexpr QChar nonbreaking_hyphen(8209); + const std::vector<QString> CONNECTION_TYPE_DOC{ + tr("Inbound Full/Block Relay: initiated by peer"), + tr("Outbound Full Relay: default"), + tr("Outbound Block Relay: does not relay transactions or addresses"), + tr("Outbound Manual: added using RPC %1 or %2/%3 configuration options") + .arg("addnode") + .arg(QString(nonbreaking_hyphen) + "addnode") + .arg(QString(nonbreaking_hyphen) + "connect"), + tr("Outbound Feeler: short-lived, for testing addresses"), + tr("Outbound Address Fetch: short-lived, for soliciting addresses")}; + const QString list{"<ul><li>" + Join(CONNECTION_TYPE_DOC, QString("</li><li>")) + "</li></ul>"}; + ui->peerConnectionTypeLabel->setToolTip(ui->peerConnectionTypeLabel->toolTip().arg(list)); ui->dataDir->setToolTip(ui->dataDir->toolTip().arg(QString(nonbreaking_hyphen) + "datadir")); ui->blocksDir->setToolTip(ui->blocksDir->toolTip().arg(QString(nonbreaking_hyphen) + "blocksdir")); ui->openDebugLogfileButton->setToolTip(ui->openDebugLogfileButton->toolTip().arg(PACKAGE_NAME)); - ui->peerConnectionTypeLabel->setToolTip(ui->peerConnectionTypeLabel->toolTip().arg("addnode").arg(QString(nonbreaking_hyphen) + "addnode").arg(QString(nonbreaking_hyphen) + "connect")); if (platformStyle->getImagesOnButtons()) { ui->openDebugLogfileButton->setIcon(platformStyle->SingleColorIcon(":/icons/export")); @@ -1108,7 +1120,7 @@ void RPCConsole::updateDetailWidget() ui->timeoffset->setText(GUIUtil::formatTimeOffset(stats->nodeStats.nTimeOffset)); ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion)); ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); - ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type)); + ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type, stats->nodeStats.fRelayTxes)); ui->peerNetwork->setText(GUIUtil::NetworkToQString(stats->nodeStats.m_network)); if (stats->nodeStats.m_permissionFlags == PF_NONE) { ui->peerPermissions->setText(tr("N/A")); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 9eedd8cf75..611f3584c3 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -55,7 +55,7 @@ int getIndexForConfTarget(int target) { } SendCoinsDialog::SendCoinsDialog(const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::SendCoinsDialog), clientModel(nullptr), model(nullptr), diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 6c110f0688..2e7df60574 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -19,7 +19,7 @@ #include <QClipboard> SignVerifyMessageDialog::SignVerifyMessageDialog(const PlatformStyle *_platformStyle, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::SignVerifyMessageDialog), model(nullptr), platformStyle(_platformStyle) diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 5796a6ef56..2292c01d6a 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -18,6 +18,8 @@ #include <util/system.h> #include <util/translation.h> +#include <functional> + #include <QApplication> #include <QCloseEvent> #include <QPainter> diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 24d7735447..0d9928d363 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -33,7 +33,7 @@ static RPCHelpMan rpcNestedTest_rpc() } static const CRPCCommand vRPCCommands[] = { - {"test", "rpcNestedTest", &rpcNestedTest_rpc, {"arg1", "arg2", "arg3"}}, + {"test", &rpcNestedTest_rpc}, }; void RPCNestedTests::rpcNestedTests() diff --git a/src/qt/transactiondescdialog.cpp b/src/qt/transactiondescdialog.cpp index b547bf7645..6f31d8463f 100644 --- a/src/qt/transactiondescdialog.cpp +++ b/src/qt/transactiondescdialog.cpp @@ -11,7 +11,7 @@ #include <QModelIndex> TransactionDescDialog::TransactionDescDialog(const QModelIndex &idx, QWidget *parent) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::TransactionDescDialog) { ui->setupUi(this); diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 05b250eea9..e6d483364c 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -19,6 +19,7 @@ #include <uint256.h> #include <algorithm> +#include <functional> #include <QColor> #include <QDateTime> diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index b7f85446f4..05499b2a41 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -29,7 +29,7 @@ /** "Help message" or "About" dialog box */ HelpMessageDialog::HelpMessageDialog(QWidget *parent, bool about) : - QDialog(parent), + QDialog(parent, GUIUtil::dialog_flags), ui(new Ui::HelpMessageDialog) { ui->setupUi(this); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index cad472b43b..02254da3ce 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -29,6 +29,7 @@ #include <wallet/wallet.h> // for CRecipient #include <stdint.h> +#include <functional> #include <QDebug> #include <QMessageBox> @@ -244,7 +245,7 @@ WalletModel::SendCoinsReturn WalletModel::sendCoins(WalletModelTransaction &tran CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << *newTx; - transaction_array.append(&(ssTx[0]), ssTx.size()); + transaction_array.append((const char*)&(ssTx[0]), ssTx.size()); } // Add addresses / update labels that we've sent to the address book, @@ -514,6 +515,13 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) questionString.append("</td><td>"); questionString.append(BitcoinUnits::formatHtmlWithUnit(getOptionsModel()->getDisplayUnit(), new_fee)); questionString.append("</td></tr></table>"); + + // Display warning in the "Confirm fee bump" window if the "Coin Control Features" option is enabled + if (getOptionsModel()->getCoinControlFeatures()) { + questionString.append("<br><br>"); + questionString.append(tr("Warning: This may pay the additional fee by reducing change outputs or adding inputs, when necessary. It may add a new change output if one does not already exist. These changes may potentially leak privacy.")); + } + SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); confirmationDialog.exec(); QMessageBox::StandardButton retval = static_cast<QMessageBox::StandardButton>(confirmationDialog.result()); diff --git a/src/rest.cpp b/src/rest.cpp index 678ffdb760..f872f6e59d 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -179,7 +179,7 @@ static bool rest_headers(const util::Ref& context, { LOCK(cs_main); tip = ::ChainActive().Tip(); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); while (pindex != nullptr && ::ChainActive().Contains(pindex)) { headers.push_back(pindex); if (headers.size() == (unsigned long)count) @@ -247,7 +247,7 @@ static bool rest_block(HTTPRequest* req, { LOCK(cs_main); tip = ::ChainActive().Tip(); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7a336c1ad6..a83b45ae1d 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -843,7 +843,7 @@ static RPCHelpMan getblockheader() const CBlockIndex* tip; { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); tip = ::ChainActive().Tip(); } @@ -967,7 +967,7 @@ static RPCHelpMan getblock() const CBlockIndex* tip; { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); tip = ::ChainActive().Tip(); if (!pblockindex) { @@ -1164,7 +1164,7 @@ static RPCHelpMan gettxout() } } - const CBlockIndex* pindex = LookupBlockIndex(coins_view->GetBestBlock()); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(coins_view->GetBestBlock()); ret.pushKV("bestblock", pindex->GetBlockHash().GetHex()); if (coin.nHeight == MEMPOOL_HEIGHT) { ret.pushKV("confirmations", 0); @@ -1557,7 +1557,7 @@ static RPCHelpMan preciousblock() { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1595,7 +1595,7 @@ static RPCHelpMan invalidateblock() CBlockIndex* pblockindex; { LOCK(cs_main); - pblockindex = LookupBlockIndex(hash); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1603,7 +1603,7 @@ static RPCHelpMan invalidateblock() InvalidateBlock(state, Params(), pblockindex); if (state.IsValid()) { - ActivateBestChain(state, Params()); + ::ChainstateActive().ActivateBestChain(state, Params()); } if (!state.IsValid()) { @@ -1634,7 +1634,7 @@ static RPCHelpMan reconsiderblock() { LOCK(cs_main); - CBlockIndex* pblockindex = LookupBlockIndex(hash); + CBlockIndex* pblockindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1643,7 +1643,7 @@ static RPCHelpMan reconsiderblock() } BlockValidationState state; - ActivateBestChain(state, Params()); + ::ChainstateActive().ActivateBestChain(state, Params()); if (!state.IsValid()) { throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); @@ -1689,7 +1689,7 @@ static RPCHelpMan getchaintxstats() } else { uint256 hash(ParseHashV(request.params[1], "blockhash")); LOCK(cs_main); - pindex = LookupBlockIndex(hash); + pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -1867,7 +1867,7 @@ static RPCHelpMan getblockstats() pindex = ::ChainActive()[height]; } else { const uint256 hash(ParseHashV(request.params[0], "hash_or_height")); - pindex = LookupBlockIndex(hash); + pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (!pindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -2330,7 +2330,7 @@ static RPCHelpMan getblockfilter() bool block_was_connected; { LOCK(cs_main); - block_index = LookupBlockIndex(block_hash); + block_index = g_chainman.m_blockman.LookupBlockIndex(block_hash); if (!block_index) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -2440,7 +2440,7 @@ static RPCHelpMan dumptxoutset() } pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); - tip = LookupBlockIndex(stats.hashBlock); + tip = g_chainman.m_blockman.LookupBlockIndex(stats.hashBlock); CHECK_NONFATAL(tip); } @@ -2480,41 +2480,41 @@ void RegisterBlockchainRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "blockchain", "getblockchaininfo", &getblockchaininfo, {} }, - { "blockchain", "getchaintxstats", &getchaintxstats, {"nblocks", "blockhash"} }, - { "blockchain", "getblockstats", &getblockstats, {"hash_or_height", "stats"} }, - { "blockchain", "getbestblockhash", &getbestblockhash, {} }, - { "blockchain", "getblockcount", &getblockcount, {} }, - { "blockchain", "getblock", &getblock, {"blockhash","verbosity|verbose"} }, - { "blockchain", "getblockhash", &getblockhash, {"height"} }, - { "blockchain", "getblockheader", &getblockheader, {"blockhash","verbose"} }, - { "blockchain", "getchaintips", &getchaintips, {} }, - { "blockchain", "getdifficulty", &getdifficulty, {} }, - { "blockchain", "getmempoolancestors", &getmempoolancestors, {"txid","verbose"} }, - { "blockchain", "getmempooldescendants", &getmempooldescendants, {"txid","verbose"} }, - { "blockchain", "getmempoolentry", &getmempoolentry, {"txid"} }, - { "blockchain", "getmempoolinfo", &getmempoolinfo, {} }, - { "blockchain", "getrawmempool", &getrawmempool, {"verbose", "mempool_sequence"} }, - { "blockchain", "gettxout", &gettxout, {"txid","n","include_mempool"} }, - { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, {"hash_type"} }, - { "blockchain", "pruneblockchain", &pruneblockchain, {"height"} }, - { "blockchain", "savemempool", &savemempool, {} }, - { "blockchain", "verifychain", &verifychain, {"checklevel","nblocks"} }, - - { "blockchain", "preciousblock", &preciousblock, {"blockhash"} }, - { "blockchain", "scantxoutset", &scantxoutset, {"action", "scanobjects"} }, - { "blockchain", "getblockfilter", &getblockfilter, {"blockhash", "filtertype"} }, +{ // category actor (function) + // --------------------- ------------------------ + { "blockchain", &getblockchaininfo, }, + { "blockchain", &getchaintxstats, }, + { "blockchain", &getblockstats, }, + { "blockchain", &getbestblockhash, }, + { "blockchain", &getblockcount, }, + { "blockchain", &getblock, }, + { "blockchain", &getblockhash, }, + { "blockchain", &getblockheader, }, + { "blockchain", &getchaintips, }, + { "blockchain", &getdifficulty, }, + { "blockchain", &getmempoolancestors, }, + { "blockchain", &getmempooldescendants, }, + { "blockchain", &getmempoolentry, }, + { "blockchain", &getmempoolinfo, }, + { "blockchain", &getrawmempool, }, + { "blockchain", &gettxout, }, + { "blockchain", &gettxoutsetinfo, }, + { "blockchain", &pruneblockchain, }, + { "blockchain", &savemempool, }, + { "blockchain", &verifychain, }, + + { "blockchain", &preciousblock, }, + { "blockchain", &scantxoutset, }, + { "blockchain", &getblockfilter, }, /* Not shown in help */ - { "hidden", "invalidateblock", &invalidateblock, {"blockhash"} }, - { "hidden", "reconsiderblock", &reconsiderblock, {"blockhash"} }, - { "hidden", "waitfornewblock", &waitfornewblock, {"timeout"} }, - { "hidden", "waitforblock", &waitforblock, {"blockhash","timeout"} }, - { "hidden", "waitforblockheight", &waitforblockheight, {"height","timeout"} }, - { "hidden", "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} }, - { "hidden", "dumptxoutset", &dumptxoutset, {"path"} }, + { "hidden", &invalidateblock, }, + { "hidden", &reconsiderblock, }, + { "hidden", &waitfornewblock, }, + { "hidden", &waitforblock, }, + { "hidden", &waitforblockheight, }, + { "hidden", &syncwithvalidationinterfacequeue, }, + { "hidden", &dumptxoutset, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 042005b9a6..d1eb849b7e 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -210,14 +210,9 @@ public: CRPCConvertTable::CRPCConvertTable() { - const unsigned int n_elem = - (sizeof(vRPCConvertParams) / sizeof(vRPCConvertParams[0])); - - for (unsigned int i = 0; i < n_elem; i++) { - members.insert(std::make_pair(vRPCConvertParams[i].methodName, - vRPCConvertParams[i].paramIdx)); - membersByName.insert(std::make_pair(vRPCConvertParams[i].methodName, - vRPCConvertParams[i].paramName)); + for (const auto& cp : vRPCConvertParams) { + members.emplace(cp.methodName, cp.paramIdx); + membersByName.emplace(cp.methodName, cp.paramName); } } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 0e96beb6db..50987a735b 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -375,7 +375,7 @@ static RPCHelpMan generateblock() LOCK(cs_main); BlockValidationState state; - if (!TestBlockValidity(state, chainparams, block, LookupBlockIndex(block.hashPrevBlock), false, false)) { + if (!TestBlockValidity(state, chainparams, ::ChainstateActive(), block, g_chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock), false, false)) { throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("TestBlockValidity failed: %s", state.ToString())); } } @@ -618,7 +618,7 @@ static RPCHelpMan getblocktemplate() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Block decode failed"); uint256 hash = block.GetHash(); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (pindex) { if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) return "duplicate"; @@ -632,7 +632,7 @@ static RPCHelpMan getblocktemplate() if (block.hashPrevBlock != pindexPrev->GetBlockHash()) return "inconclusive-not-best-prevblk"; BlockValidationState state; - TestBlockValidity(state, Params(), block, pindexPrev, false, true); + TestBlockValidity(state, Params(), ::ChainstateActive(), block, pindexPrev, false, true); return BIP22ValidationResult(state); } @@ -966,7 +966,7 @@ static RPCHelpMan submitblock() uint256 hash = block.GetHash(); { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(hash); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hash); if (pindex) { if (pindex->IsValid(BLOCK_VALID_SCRIPTS)) { return "duplicate"; @@ -979,7 +979,7 @@ static RPCHelpMan submitblock() { LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(block.hashPrevBlock); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(block.hashPrevBlock); if (pindex) { UpdateUncommittedBlockStructures(block, pindex, Params().GetConsensus()); } @@ -1023,7 +1023,7 @@ static RPCHelpMan submitheader() } { LOCK(cs_main); - if (!LookupBlockIndex(h.hashPrevBlock)) { + if (!g_chainman.m_blockman.LookupBlockIndex(h.hashPrevBlock)) { throw JSONRPCError(RPC_VERIFY_ERROR, "Must submit previous header (" + h.hashPrevBlock.GetHex() + ") first"); } } @@ -1232,24 +1232,24 @@ void RegisterMiningRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "mining", "getnetworkhashps", &getnetworkhashps, {"nblocks","height"} }, - { "mining", "getmininginfo", &getmininginfo, {} }, - { "mining", "prioritisetransaction", &prioritisetransaction, {"txid","dummy","fee_delta"} }, - { "mining", "getblocktemplate", &getblocktemplate, {"template_request"} }, - { "mining", "submitblock", &submitblock, {"hexdata","dummy"} }, - { "mining", "submitheader", &submitheader, {"hexdata"} }, +{ // category actor (function) + // --------------------- ----------------------- + { "mining", &getnetworkhashps, }, + { "mining", &getmininginfo, }, + { "mining", &prioritisetransaction, }, + { "mining", &getblocktemplate, }, + { "mining", &submitblock, }, + { "mining", &submitheader, }, - { "generating", "generatetoaddress", &generatetoaddress, {"nblocks","address","maxtries"} }, - { "generating", "generatetodescriptor", &generatetodescriptor, {"num_blocks","descriptor","maxtries"} }, - { "generating", "generateblock", &generateblock, {"output","transactions"} }, + { "generating", &generatetoaddress, }, + { "generating", &generatetodescriptor, }, + { "generating", &generateblock, }, - { "util", "estimatesmartfee", &estimatesmartfee, {"conf_target", "estimate_mode"} }, + { "util", &estimatesmartfee, }, - { "hidden", "estimaterawfee", &estimaterawfee, {"conf_target", "threshold"} }, - { "hidden", "generate", &generate, {} }, + { "hidden", &estimaterawfee, }, + { "hidden", &generate, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index b3102a236d..b75a7b8d26 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -39,13 +39,14 @@ static RPCHelpMan validateaddress() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not. If not, this is the only property returned."}, + {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not"}, {RPCResult::Type::STR, "address", "The bitcoin address validated"}, {RPCResult::Type::STR_HEX, "scriptPubKey", "The hex-encoded scriptPubKey generated by the address"}, {RPCResult::Type::BOOL, "isscript", "If the key is a script"}, {RPCResult::Type::BOOL, "iswitness", "If the address is a witness address"}, {RPCResult::Type::NUM, "witness_version", /* optional */ true, "The version number of the witness program"}, {RPCResult::Type::STR_HEX, "witness_program", /* optional */ true, "The hex value of the witness program"}, + {RPCResult::Type::STR, "error", /* optional */ true, "Error message, if any"}, } }, RPCExamples{ @@ -54,13 +55,14 @@ static RPCHelpMan validateaddress() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - CTxDestination dest = DecodeDestination(request.params[0].get_str()); - bool isValid = IsValidDestination(dest); + std::string error_msg; + CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg); + const bool isValid = IsValidDestination(dest); + CHECK_NONFATAL(isValid == error_msg.empty()); UniValue ret(UniValue::VOBJ); ret.pushKV("isvalid", isValid); - if (isValid) - { + if (isValid) { std::string currentAddress = EncodeDestination(dest); ret.pushKV("address", currentAddress); @@ -69,7 +71,10 @@ static RPCHelpMan validateaddress() UniValue detail = DescribeAddress(dest); ret.pushKVs(detail); + } else { + ret.pushKV("error", error_msg); } + return ret; }, }; @@ -694,23 +699,23 @@ void RegisterMiscRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "control", "getmemoryinfo", &getmemoryinfo, {"mode"} }, - { "control", "logging", &logging, {"include", "exclude"}}, - { "util", "validateaddress", &validateaddress, {"address"} }, - { "util", "createmultisig", &createmultisig, {"nrequired","keys","address_type"} }, - { "util", "deriveaddresses", &deriveaddresses, {"descriptor", "range"} }, - { "util", "getdescriptorinfo", &getdescriptorinfo, {"descriptor"} }, - { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, - { "util", "signmessagewithprivkey", &signmessagewithprivkey, {"privkey","message"} }, - { "util", "getindexinfo", &getindexinfo, {"index_name"} }, +{ // category actor (function) + // --------------------- ------------------------ + { "control", &getmemoryinfo, }, + { "control", &logging, }, + { "util", &validateaddress, }, + { "util", &createmultisig, }, + { "util", &deriveaddresses, }, + { "util", &getdescriptorinfo, }, + { "util", &verifymessage, }, + { "util", &signmessagewithprivkey, }, + { "util", &getindexinfo, }, /* Not shown in help */ - { "hidden", "setmocktime", &setmocktime, {"timestamp"}}, - { "hidden", "mockscheduler", &mockscheduler, {"delta_time"}}, - { "hidden", "echo", &echo, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, - { "hidden", "echojson", &echojson, {"arg0","arg1","arg2","arg3","arg4","arg5","arg6","arg7","arg8","arg9"}}, + { "hidden", &setmocktime, }, + { "hidden", &mockscheduler, }, + { "hidden", &echo, }, + { "hidden", &echojson, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index cfca8b4ad4..47d77b341a 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -941,24 +941,24 @@ void RegisterNetRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "network", "getconnectioncount", &getconnectioncount, {} }, - { "network", "ping", &ping, {} }, - { "network", "getpeerinfo", &getpeerinfo, {} }, - { "network", "addnode", &addnode, {"node","command"} }, - { "network", "disconnectnode", &disconnectnode, {"address", "nodeid"} }, - { "network", "getaddednodeinfo", &getaddednodeinfo, {"node"} }, - { "network", "getnettotals", &getnettotals, {} }, - { "network", "getnetworkinfo", &getnetworkinfo, {} }, - { "network", "setban", &setban, {"subnet", "command", "bantime", "absolute"} }, - { "network", "listbanned", &listbanned, {} }, - { "network", "clearbanned", &clearbanned, {} }, - { "network", "setnetworkactive", &setnetworkactive, {"state"} }, - { "network", "getnodeaddresses", &getnodeaddresses, {"count"} }, - - { "hidden", "addconnection", &addconnection, {"address", "connection_type"} }, - { "hidden", "addpeeraddress", &addpeeraddress, {"address", "port"} }, +{ // category actor + // --------------------- ----------------------- + { "network", &getconnectioncount, }, + { "network", &ping, }, + { "network", &getpeerinfo, }, + { "network", &addnode, }, + { "network", &disconnectnode, }, + { "network", &getaddednodeinfo, }, + { "network", &getnettotals, }, + { "network", &getnetworkinfo, }, + { "network", &setban, }, + { "network", &listbanned, }, + { "network", &clearbanned, }, + { "network", &setnetworkactive, }, + { "network", &getnodeaddresses, }, + + { "hidden", &addconnection, }, + { "hidden", &addpeeraddress, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/protocol.h b/src/rpc/protocol.h index c8ceb2c186..fc00a1efad 100644 --- a/src/rpc/protocol.h +++ b/src/rpc/protocol.h @@ -79,6 +79,7 @@ enum RPCErrorCode RPC_WALLET_ALREADY_UNLOCKED = -17, //!< Wallet is already unlocked RPC_WALLET_NOT_FOUND = -18, //!< Invalid wallet specified RPC_WALLET_NOT_SPECIFIED = -19, //!< No wallet specified (error when there are multiple wallets loaded) + RPC_WALLET_ALREADY_LOADED = -35, //!< This same wallet is already loaded //! Backwards compatible aliases RPC_WALLET_INVALID_ACCOUNT_NAME = RPC_WALLET_INVALID_LABEL_NAME, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index e94d554e3c..ac42404470 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -54,7 +54,7 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& LOCK(cs_main); entry.pushKV("blockhash", hashBlock.GetHex()); - CBlockIndex* pindex = LookupBlockIndex(hashBlock); + CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(hashBlock); if (pindex) { if (::ChainActive().Contains(pindex)) { entry.pushKV("confirmations", 1 + ::ChainActive().Height() - pindex->nHeight); @@ -178,7 +178,7 @@ static RPCHelpMan getrawtransaction() LOCK(cs_main); uint256 blockhash = ParseHashV(request.params[2], "parameter 3"); - blockindex = LookupBlockIndex(blockhash); + blockindex = g_chainman.m_blockman.LookupBlockIndex(blockhash); if (!blockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block hash not found"); } @@ -260,7 +260,7 @@ static RPCHelpMan gettxoutproof() if (!request.params[1].isNull()) { LOCK(cs_main); hashBlock = ParseHashV(request.params[1], "blockhash"); - pblockindex = LookupBlockIndex(hashBlock); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } @@ -290,7 +290,7 @@ static RPCHelpMan gettxoutproof() if (!tx || hashBlock.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); } - pblockindex = LookupBlockIndex(hashBlock); + pblockindex = g_chainman.m_blockman.LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); } @@ -350,7 +350,7 @@ static RPCHelpMan verifytxoutproof() LOCK(cs_main); - const CBlockIndex* pindex = LookupBlockIndex(merkleBlock.header.GetHash()); + const CBlockIndex* pindex = g_chainman.m_blockman.LookupBlockIndex(merkleBlock.header.GetHash()); if (!pindex || !::ChainActive().Contains(pindex) || pindex->nTx == 0) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); } @@ -816,10 +816,11 @@ static RPCHelpMan sendrawtransaction() { return RPCHelpMan{"sendrawtransaction", "\nSubmit a raw transaction (serialized, hex-encoded) to local node and network.\n" - "\nNote that the transaction will be sent unconditionally to all peers, so using this\n" + "\nThe transaction will be sent unconditionally to all peers, so using sendrawtransaction\n" "for manual rebroadcast may degrade privacy by leaking the transaction's origin, as\n" "nodes will normally not rebroadcast non-wallet transactions already in their mempool.\n" - "\nAlso see createrawtransaction and signrawtransactionwithkey calls.\n", + "\nA specific exception, RPC_TRANSACTION_ALREADY_IN_CHAIN, may throw if the transaction cannot be added to the mempool.\n" + "\nRelated RPCs: createrawtransaction, signrawtransactionwithkey\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, {"maxfeerate", RPCArg::Type::AMOUNT, /* default */ FormatMoney(DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK()), @@ -1344,7 +1345,7 @@ static RPCHelpMan combinepsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << merged_psbt; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1483,7 +1484,7 @@ static RPCHelpMan createpsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1552,7 +1553,7 @@ static RPCHelpMan converttopsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1643,7 +1644,7 @@ static RPCHelpMan utxoupdatepsbt() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1739,7 +1740,7 @@ static RPCHelpMan joinpsbts() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << shuffled_psbt; - return EncodeBase64(MakeUCharSpan(ssTx)); + return EncodeBase64(ssTx); }, }; } @@ -1859,27 +1860,27 @@ void RegisterRawTransactionRPCCommands(CRPCTable &t) { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "rawtransactions", "getrawtransaction", &getrawtransaction, {"txid","verbose","blockhash"} }, - { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, - { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, - { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, - { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","maxfeerate"} }, - { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, - { "rawtransactions", "signrawtransactionwithkey", &signrawtransactionwithkey, {"hexstring","privkeys","prevtxs","sighashtype"} }, - { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","maxfeerate"} }, - { "rawtransactions", "decodepsbt", &decodepsbt, {"psbt"} }, - { "rawtransactions", "combinepsbt", &combinepsbt, {"txs"} }, - { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, - { "rawtransactions", "createpsbt", &createpsbt, {"inputs","outputs","locktime","replaceable"} }, - { "rawtransactions", "converttopsbt", &converttopsbt, {"hexstring","permitsigdata","iswitness"} }, - { "rawtransactions", "utxoupdatepsbt", &utxoupdatepsbt, {"psbt", "descriptors"} }, - { "rawtransactions", "joinpsbts", &joinpsbts, {"txs"} }, - { "rawtransactions", "analyzepsbt", &analyzepsbt, {"psbt"} }, - - { "blockchain", "gettxoutproof", &gettxoutproof, {"txids", "blockhash"} }, - { "blockchain", "verifytxoutproof", &verifytxoutproof, {"proof"} }, +{ // category actor (function) + // --------------------- ----------------------- + { "rawtransactions", &getrawtransaction, }, + { "rawtransactions", &createrawtransaction, }, + { "rawtransactions", &decoderawtransaction, }, + { "rawtransactions", &decodescript, }, + { "rawtransactions", &sendrawtransaction, }, + { "rawtransactions", &combinerawtransaction, }, + { "rawtransactions", &signrawtransactionwithkey, }, + { "rawtransactions", &testmempoolaccept, }, + { "rawtransactions", &decodepsbt, }, + { "rawtransactions", &combinepsbt, }, + { "rawtransactions", &finalizepsbt, }, + { "rawtransactions", &createpsbt, }, + { "rawtransactions", &converttopsbt, }, + { "rawtransactions", &utxoupdatepsbt, }, + { "rawtransactions", &joinpsbts, }, + { "rawtransactions", &analyzepsbt, }, + + { "blockchain", &gettxoutproof, }, + { "blockchain", &verifytxoutproof, }, }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index f32d9abac6..9a9b3713f3 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -144,8 +144,13 @@ static RPCHelpMan help() [&](const RPCHelpMan& self, const JSONRPCRequest& jsonRequest) -> UniValue { std::string strCommand; - if (jsonRequest.params.size() > 0) + if (jsonRequest.params.size() > 0) { strCommand = jsonRequest.params[0].get_str(); + } + if (strCommand == "dump_all_command_conversions") { + // Used for testing only, undocumented + return tableRPC.dumpArgMap(); + } return tableRPC.help(strCommand, jsonRequest); }, @@ -244,13 +249,13 @@ static RPCHelpMan getrpcinfo() // clang-format off static const CRPCCommand vRPCCommands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- +{ // category actor (function) + // --------------------- ----------------------- /* Overall control/query calls */ - { "control", "getrpcinfo", &getrpcinfo, {} }, - { "control", "help", &help, {"command"} }, - { "control", "stop", &stop, {"wait"} }, - { "control", "uptime", &uptime, {} }, + { "control", &getrpcinfo, }, + { "control", &help, }, + { "control", &stop, }, + { "control", &uptime, }, }; // clang-format on @@ -479,6 +484,18 @@ std::vector<std::string> CRPCTable::listCommands() const return commandList; } +UniValue CRPCTable::dumpArgMap() const +{ + UniValue ret{UniValue::VARR}; + for (const auto& cmd : mapCommands) { + for (const auto& c : cmd.second) { + const auto help = RpcMethodFnType(c->unique_id)(); + help.AppendArgMap(ret); + } + } + return ret; +} + void RPCSetTimerInterfaceIfUnset(RPCTimerInterface *iface) { if (!timerInterface) diff --git a/src/rpc/server.h b/src/rpc/server.h index 040192b455..fe5a791e1e 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -103,7 +103,7 @@ public: } //! Simplified constructor taking plain RpcMethodFnType function pointer. - CRPCCommand(std::string category, std::string name_in, RpcMethodFnType fn, std::vector<std::string> args_in) + CRPCCommand(std::string category, RpcMethodFnType fn) : CRPCCommand( category, fn().m_name, @@ -111,8 +111,6 @@ public: fn().GetArgNames(), intptr_t(fn)) { - CHECK_NONFATAL(fn().m_name == name_in); - CHECK_NONFATAL(fn().GetArgNames() == args_in); } std::string category; @@ -147,6 +145,10 @@ public: */ std::vector<std::string> listCommands() const; + /** + * Return all named arguments that need to be converted by the client from string to another JSON type + */ + UniValue dumpArgMap() const; /** * Appends a CRPCCommand to the dispatch table. diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 31072114da..bfdba5253c 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -549,6 +549,24 @@ std::string RPCHelpMan::ToString() const return ret; } +void RPCHelpMan::AppendArgMap(UniValue& arr) const +{ + for (int i{0}; i < int(m_args.size()); ++i) { + const auto& arg = m_args.at(i); + std::vector<std::string> arg_names; + boost::split(arg_names, arg.m_names, boost::is_any_of("|")); + for (const auto& arg_name : arg_names) { + UniValue map{UniValue::VARR}; + map.push_back(m_name); + map.push_back(i); + map.push_back(arg_name); + map.push_back(arg.m_type == RPCArg::Type::STR || + arg.m_type == RPCArg::Type::STR_HEX); + arr.push_back(map); + } + } +} + std::string RPCArg::GetFirstName() const { return m_names.substr(0, m_names.find("|")); diff --git a/src/rpc/util.h b/src/rpc/util.h index 942c243718..444a013ca1 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -336,6 +336,8 @@ public: RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples, RPCMethodImpl fun); std::string ToString() const; + /** Append the named args that need to be converted from string to another JSON type */ + void AppendArgMap(UniValue& arr) const; UniValue HandleRequest(const JSONRPCRequest& request) { Check(request); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 7c361bf26f..b3ee23f139 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -7,6 +7,7 @@ #include <random.h> #include <assert.h> +#include <functional> #include <utility> CScheduler::CScheduler() diff --git a/src/scheduler.h b/src/scheduler.h index d7fe00d1b4..9eec8c0fa0 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -9,6 +9,7 @@ #include <functional> #include <list> #include <map> +#include <thread> #include <sync.h> @@ -35,6 +36,8 @@ public: CScheduler(); ~CScheduler(); + std::thread m_service_thread; + typedef std::function<void()> Function; /** Call func at/after time t */ @@ -62,8 +65,7 @@ public: void MockForward(std::chrono::seconds delta_seconds); /** - * Services the queue 'forever'. Should be run in a thread, - * and interrupted using boost::interrupt_thread + * Services the queue 'forever'. Should be run in a thread. */ void serviceQueue(); @@ -72,12 +74,14 @@ public: { WITH_LOCK(newTaskMutex, stopRequested = true); newTaskScheduled.notify_all(); + if (m_service_thread.joinable()) m_service_thread.join(); } /** Tell any threads running serviceQueue to stop when there is no work left to be done */ void StopWhenDrained() { WITH_LOCK(newTaskMutex, stopWhenEmpty = true); newTaskScheduled.notify_all(); + if (m_service_thread.joinable()) m_service_thread.join(); } /** diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 2bfe206597..cf47d37e70 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -11,6 +11,8 @@ #include <util/system.h> #include <cuckoocache.h> + +#include <boost/thread/lock_types.hpp> #include <boost/thread/shared_mutex.hpp> namespace { diff --git a/src/streams.h b/src/streams.h index fce411d3df..e78da31cbc 100644 --- a/src/streams.h +++ b/src/streams.h @@ -6,17 +6,19 @@ #ifndef BITCOIN_STREAMS_H #define BITCOIN_STREAMS_H -#include <support/allocators/zeroafterfree.h> #include <serialize.h> +#include <span.h> +#include <support/allocators/zeroafterfree.h> #include <algorithm> #include <assert.h> #include <ios> #include <limits> +#include <optional> #include <stdint.h> #include <stdio.h> -#include <string> #include <string.h> +#include <string> #include <utility> #include <vector> @@ -202,14 +204,14 @@ public: class CDataStream { protected: - typedef CSerializeData vector_type; + using vector_type = SerializeData; vector_type vch; - unsigned int nReadPos; + unsigned int nReadPos{0}; int nType; int nVersion; -public: +public: typedef vector_type::allocator_type allocator_type; typedef vector_type::size_type size_type; typedef vector_type::difference_type difference_type; @@ -221,62 +223,22 @@ public: typedef vector_type::reverse_iterator reverse_iterator; explicit CDataStream(int nTypeIn, int nVersionIn) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const_iterator pbegin, const_iterator pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const char* pbegin, const char* pend, int nTypeIn, int nVersionIn) : vch(pbegin, pend) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const vector_type& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } - - CDataStream(const std::vector<char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } + : nType{nTypeIn}, + nVersion{nVersionIn} {} - CDataStream(const std::vector<unsigned char>& vchIn, int nTypeIn, int nVersionIn) : vch(vchIn.begin(), vchIn.end()) - { - Init(nTypeIn, nVersionIn); - } + explicit CDataStream(Span<const uint8_t> sp, int nTypeIn, int nVersionIn) + : vch(sp.data(), sp.data() + sp.size()), + nType{nTypeIn}, + nVersion{nVersionIn} {} template <typename... Args> CDataStream(int nTypeIn, int nVersionIn, Args&&... args) + : nType{nTypeIn}, + nVersion{nVersionIn} { - Init(nTypeIn, nVersionIn); ::SerializeMany(*this, std::forward<Args>(args)...); } - void Init(int nTypeIn, int nVersionIn) - { - nReadPos = 0; - nType = nTypeIn; - nVersion = nVersionIn; - } - - CDataStream& operator+=(const CDataStream& b) - { - vch.insert(vch.end(), b.begin(), b.end()); - return *this; - } - - friend CDataStream operator+(const CDataStream& a, const CDataStream& b) - { - CDataStream ret = a; - ret += b; - return (ret); - } - std::string str() const { return (std::string(begin(), end())); @@ -297,12 +259,12 @@ public: const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } reference operator[](size_type pos) { return vch[pos + nReadPos]; } void clear() { vch.clear(); nReadPos = 0; } - iterator insert(iterator it, const char x=char()) { return vch.insert(it, x); } - void insert(iterator it, size_type n, const char x) { vch.insert(it, n, x); } + iterator insert(iterator it, const uint8_t x) { return vch.insert(it, x); } + void insert(iterator it, size_type n, const uint8_t x) { vch.insert(it, n, x); } value_type* data() { return vch.data() + nReadPos; } const value_type* data() const { return vch.data() + nReadPos; } - void insert(iterator it, std::vector<char>::const_iterator first, std::vector<char>::const_iterator last) + void insert(iterator it, std::vector<uint8_t>::const_iterator first, std::vector<uint8_t>::const_iterator last) { if (last == first) return; assert(last - first > 0); @@ -373,12 +335,17 @@ public: nReadPos = 0; } - bool Rewind(size_type n) + bool Rewind(std::optional<size_type> n = std::nullopt) { + // Total rewind if no size is passed + if (!n) { + nReadPos = 0; + return true; + } // Rewind by n characters if the buffer hasn't been compacted yet - if (n > nReadPos) + if (*n > nReadPos) return false; - nReadPos -= n; + nReadPos -= *n; return true; } @@ -462,11 +429,6 @@ public: return (*this); } - void GetAndClear(CSerializeData &d) { - d.insert(d.end(), begin(), end()); - clear(); - } - /** * XOR the contents of this stream with a certain key. * diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index c7ed5ef308..418f0ee656 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -42,7 +42,7 @@ struct zero_after_free_allocator : public std::allocator<T> { } }; -// Byte-vector that clears its contents before deletion. -typedef std::vector<char, zero_after_free_allocator<char> > CSerializeData; +/** Byte-vector that clears its contents before deletion. */ +using SerializeData = std::vector<uint8_t, zero_after_free_allocator<uint8_t>>; #endif // BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H diff --git a/src/sync.cpp b/src/sync.cpp index acfbe8fe29..a2b62c2286 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -13,8 +13,6 @@ #include <util/strencodings.h> #include <util/threadnames.h> -#include <boost/thread/mutex.hpp> - #include <map> #include <mutex> #include <set> @@ -224,7 +222,6 @@ template void EnterCritical(const char*, const char*, int, Mutex*, bool); template void EnterCritical(const char*, const char*, int, RecursiveMutex*, bool); template void EnterCritical(const char*, const char*, int, std::mutex*, bool); template void EnterCritical(const char*, const char*, int, std::recursive_mutex*, bool); -template void EnterCritical(const char*, const char*, int, boost::mutex*, bool); void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line) { diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 00c4bdc14e..633a95ce96 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -178,7 +178,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) const CBlockIndex* block_index; { LOCK(cs_main); - block_index = LookupBlockIndex(block->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(block->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); @@ -196,7 +196,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) const CBlockIndex* block_index; { LOCK(cs_main); - block_index = LookupBlockIndex(block->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(block->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); @@ -210,7 +210,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) const CBlockIndex* block_index; { LOCK(cs_main); - block_index = LookupBlockIndex(block->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(block->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); @@ -231,14 +231,14 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) { LOCK(cs_main); - block_index = LookupBlockIndex(chainA[i]->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(chainA[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); CheckFilterLookups(filter_index, block_index, chainA_last_header); { LOCK(cs_main); - block_index = LookupBlockIndex(chainB[i]->GetHash()); + block_index = g_chainman.m_blockman.LookupBlockIndex(chainB[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); CheckFilterLookups(filter_index, block_index, chainB_last_header); diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 736c260eeb..5a98558240 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -42,11 +42,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; - std::vector<unsigned char> vch = ParseHex("03614e9b050000000000000001"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<uint8_t> expected = ParseHex("03614e9b050000000000000001"); BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); @@ -72,11 +68,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; - std::vector<unsigned char> vch = ParseHex("03ce4299050000000100008001"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<uint8_t> expected = ParseHex("03ce4299050000000100008001"); BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); } @@ -96,11 +88,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_key) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; - std::vector<unsigned char> vch = ParseHex("038fc16b080000000000000001"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<unsigned char> expected = ParseHex("038fc16b080000000000000001"); BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); } @@ -352,11 +340,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_3_and_serialize) CDataStream merkleStream(SER_NETWORK, PROTOCOL_VERSION); merkleStream << merkleBlock; - std::vector<unsigned char> vch = ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630100000001b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f19630101"); - std::vector<char> expected(vch.size()); - - for (unsigned int i = 0; i < vch.size(); i++) - expected[i] = (char)vch[i]; + std::vector<uint8_t> expected = ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630100000001b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f19630101"); BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), merkleStream.begin(), merkleStream.end()); } diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 5dbf07b420..21921375b3 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -10,7 +10,6 @@ #include <util/time.h> #include <boost/test/unit_test.hpp> -#include <boost/thread/thread.hpp> #include <atomic> #include <condition_variable> @@ -148,10 +147,7 @@ typedef CCheckQueue<FrozenCleanupCheck> FrozenCleanup_Queue; static void Correct_Queue_range(std::vector<size_t> range) { auto small_queue = MakeUnique<Correct_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{small_queue->Thread();}); - } + small_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); // Make vChecks here to save on malloc (this test can be slow...) std::vector<FakeCheckCheckCompletion> vChecks; for (const size_t i : range) { @@ -168,8 +164,7 @@ static void Correct_Queue_range(std::vector<size_t> range) BOOST_REQUIRE_EQUAL(FakeCheckCheckCompletion::n_calls, i); } } - tg.interrupt_all(); - tg.join_all(); + small_queue->StopWorkerThreads(); } /** Test that 0 checks is correct @@ -212,11 +207,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Correct_Random) BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) { auto fail_queue = MakeUnique<Failing_Queue>(QUEUE_BATCH_SIZE); - - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{fail_queue->Thread();}); - } + fail_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); for (size_t i = 0; i < 1001; ++i) { CCheckQueueControl<FailingCheck> control(fail_queue.get()); @@ -237,18 +228,14 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) BOOST_REQUIRE(success); } } - tg.interrupt_all(); - tg.join_all(); + fail_queue->StopWorkerThreads(); } // Test that a block validation which fails does not interfere with // future blocks, ie, the bad state is cleared. BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) { auto fail_queue = MakeUnique<Failing_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{fail_queue->Thread();}); - } + fail_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); for (auto times = 0; times < 10; ++times) { for (const bool end_fails : {true, false}) { @@ -263,8 +250,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) BOOST_REQUIRE(r != end_fails); } } - tg.interrupt_all(); - tg.join_all(); + fail_queue->StopWorkerThreads(); } // Test that unique checks are actually all called individually, rather than @@ -273,11 +259,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) { auto queue = MakeUnique<Unique_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{queue->Thread();}); - - } + queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); size_t COUNT = 100000; size_t total = COUNT; @@ -300,8 +282,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) } BOOST_REQUIRE(r); } - tg.interrupt_all(); - tg.join_all(); + queue->StopWorkerThreads(); } @@ -313,10 +294,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) { auto queue = MakeUnique<Memory_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{queue->Thread();}); - } + queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); for (size_t i = 0; i < 1000; ++i) { size_t total = i; { @@ -335,8 +313,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) } BOOST_REQUIRE_EQUAL(MemoryCheck::fake_allocated_memory, 0U); } - tg.interrupt_all(); - tg.join_all(); + queue->StopWorkerThreads(); } // Test that a new verification cannot occur until all checks @@ -344,11 +321,8 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) { auto queue = MakeUnique<FrozenCleanup_Queue>(QUEUE_BATCH_SIZE); - boost::thread_group tg; bool fails = false; - for (auto x = 0; x < SCRIPT_CHECK_THREADS; ++x) { - tg.create_thread([&]{queue->Thread();}); - } + queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); std::thread t0([&]() { CCheckQueueControl<FrozenCleanupCheck> control(queue.get()); std::vector<FrozenCleanupCheck> vChecks(1); @@ -367,7 +341,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) } // Try to get control of the queue a bunch of times for (auto x = 0; x < 100 && !fails; ++x) { - fails = queue->ControlMutex.try_lock(); + fails = queue->m_control_mutex.try_lock(); } { // Unfreeze (we need lock n case of spurious wakeup) @@ -378,9 +352,8 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) FrozenCleanupCheck::cv.notify_one(); // Wait for control to finish t0.join(); - tg.interrupt_all(); - tg.join_all(); BOOST_REQUIRE(!fails); + queue->StopWorkerThreads(); } @@ -389,11 +362,11 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) { auto queue = MakeUnique<Standard_Queue>(QUEUE_BATCH_SIZE); { - boost::thread_group tg; + std::vector<std::thread> tg; std::atomic<int> nThreads {0}; std::atomic<int> fails {0}; for (size_t i = 0; i < 3; ++i) { - tg.create_thread( + tg.emplace_back( [&]{ CCheckQueueControl<FakeCheck> control(queue.get()); // While sleeping, no other thread should execute to this point @@ -402,11 +375,13 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) fails += observed != nThreads; }); } - tg.join_all(); + for (auto& thread: tg) { + if (thread.joinable()) thread.join(); + } BOOST_REQUIRE_EQUAL(fails, 0); } { - boost::thread_group tg; + std::vector<std::thread> tg; std::mutex m; std::condition_variable cv; bool has_lock{false}; @@ -415,7 +390,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) bool done_ack{false}; { std::unique_lock<std::mutex> l(m); - tg.create_thread([&]{ + tg.emplace_back([&]{ CCheckQueueControl<FakeCheck> control(queue.get()); std::unique_lock<std::mutex> ll(m); has_lock = true; @@ -431,7 +406,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) cv.wait(l, [&](){return has_lock;}); bool fails = false; for (auto x = 0; x < 100 && !fails; ++x) { - fails = queue->ControlMutex.try_lock(); + fails = queue->m_control_mutex.try_lock(); } has_tried = true; cv.notify_one(); @@ -441,8 +416,9 @@ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) cv.notify_one(); BOOST_REQUIRE(!fails); } - tg.join_all(); + for (auto& thread: tg) { + if (thread.joinable()) thread.join(); + } } } BOOST_AUTO_TEST_SUITE_END() - diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 3a951d28ae..75c7e47e64 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> +#include <boost/thread/lock_types.hpp> +#include <boost/thread/shared_mutex.hpp> #include <cuckoocache.h> #include <deque> #include <random.h> diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp index 1f122082b2..0e1c44cded 100644 --- a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp +++ b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp @@ -45,18 +45,24 @@ FUZZ_TARGET(crypto_chacha20_poly1305_aead) assert(ok); }, [&] { + if (AdditionOverflow(seqnr_payload, static_cast<uint64_t>(1))) { + return; + } seqnr_payload += 1; aad_pos += CHACHA20_POLY1305_AEAD_AAD_LEN; if (aad_pos + CHACHA20_POLY1305_AEAD_AAD_LEN > CHACHA20_ROUND_OUTPUT) { aad_pos = 0; + if (AdditionOverflow(seqnr_aad, static_cast<uint64_t>(1))) { + return; + } seqnr_aad += 1; } }, [&] { - seqnr_payload = fuzzed_data_provider.ConsumeIntegral<int>(); + seqnr_payload = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); }, [&] { - seqnr_aad = fuzzed_data_provider.ConsumeIntegral<int>(); + seqnr_aad = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); }, [&] { is_encrypt = fuzzed_data_provider.ConsumeBool(); diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp index 207ee586bc..95597bf082 100644 --- a/src/test/fuzz/load_external_block_file.cpp +++ b/src/test/fuzz/load_external_block_file.cpp @@ -27,5 +27,5 @@ FUZZ_TARGET_INIT(load_external_block_file, initialize_load_external_block_file) return; } FlatFilePos flat_file_pos; - LoadExternalBlockFile(Params(), fuzzed_block_file, fuzzed_data_provider.ConsumeBool() ? &flat_file_pos : nullptr); + ::ChainstateActive().LoadExternalBlockFile(Params(), fuzzed_block_file, fuzzed_data_provider.ConsumeBool() ? &flat_file_pos : nullptr); } diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index 02beb6eb37..c4348495bf 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -43,7 +43,10 @@ FUZZ_TARGET_INIT(pow, initialize_pow) current_block.nHeight = current_height; } if (fuzzed_data_provider.ConsumeBool()) { - current_block.nTime = fixed_time + current_height * consensus_params.nPowTargetSpacing; + const uint32_t seconds = current_height * consensus_params.nPowTargetSpacing; + if (!AdditionOverflow(fixed_time, seconds)) { + current_block.nTime = fixed_time + seconds; + } } if (fuzzed_data_provider.ConsumeBool()) { current_block.nBits = fixed_bits; diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 5d6a33d7c2..e7cc0f5297 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -60,10 +60,12 @@ void fuzz_target(const std::vector<uint8_t>& buffer, const std::string& LIMIT_TO return; } CNode& p2p_node = *ConsumeNodeAsUniquePtr(fuzzed_data_provider).release(); - FillNode(fuzzed_data_provider, p2p_node); - p2p_node.fSuccessfullyConnected = true; + + const bool successfully_connected{true}; + p2p_node.fSuccessfullyConnected = successfully_connected; connman.AddTestNode(p2p_node); g_setup->m_node.peerman->InitializeNode(&p2p_node); + FillNode(fuzzed_data_provider, p2p_node, /* init_version */ successfully_connected); const auto mock_time = ConsumeTime(fuzzed_data_provider); SetMockTime(mock_time); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index d0d0e19694..810f0aac92 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -45,11 +45,12 @@ FUZZ_TARGET_INIT(process_messages, initialize_process_messages) for (int i = 0; i < num_peers_to_add; ++i) { peers.push_back(ConsumeNodeAsUniquePtr(fuzzed_data_provider, i).release()); CNode& p2p_node = *peers.back(); - FillNode(fuzzed_data_provider, p2p_node); - p2p_node.fSuccessfullyConnected = true; + const bool successfully_connected{true}; + p2p_node.fSuccessfullyConnected = successfully_connected; p2p_node.fPauseSend = false; g_setup->m_node.peerman->InitializeNode(&p2p_node); + FillNode(fuzzed_data_provider, p2p_node, /* init_version */ successfully_connected); connman.AddTestNode(p2p_node); } diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index d883426c81..7fadf36f98 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -154,13 +154,13 @@ FUZZ_TARGET_INIT(script, initialize_script) { WitnessUnknown witness_unknown_1{}; - witness_unknown_1.version = fuzzed_data_provider.ConsumeIntegral<int>(); + witness_unknown_1.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); witness_unknown_1.length = witness_unknown_program_1.size(); std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown_1.program); WitnessUnknown witness_unknown_2{}; - witness_unknown_2.version = fuzzed_data_provider.ConsumeIntegral<int>(); + witness_unknown_2.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const std::vector<uint8_t> witness_unknown_program_2 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); witness_unknown_2.length = witness_unknown_program_2.size(); std::copy(witness_unknown_program_2.begin(), witness_unknown_program_2.end(), witness_unknown_2.program); diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index 47b38b6d23..3621702e45 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -54,7 +54,7 @@ FUZZ_TARGET(system) if (args_manager.GetArgFlags(argument_name) != nullopt) { return; } - args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), options_category); + args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>() & ~ArgsManager::COMMAND, options_category); }, [&] { // Avoid hitting: diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp new file mode 100644 index 0000000000..0a541e4186 --- /dev/null +++ b/src/test/fuzz/util.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/util.h> +#include <version.h> + +void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, bool init_version) noexcept +{ + const ServiceFlags remote_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); + const NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); + const int32_t version = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(MIN_PEER_PROTO_VERSION, std::numeric_limits<int32_t>::max()); + const bool filter_txs = fuzzed_data_provider.ConsumeBool(); + + node.nServices = remote_services; + node.m_permissionFlags = permission_flags; + if (init_version) { + node.nVersion = version; + node.SetCommonVersion(std::min(version, PROTOCOL_VERSION)); + } + if (node.m_tx_relay != nullptr) { + LOCK(node.m_tx_relay->cs_filter); + node.m_tx_relay->fRelayTxes = filter_txs; + } +} diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 7796f77cc6..7a2dcfe84a 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -61,7 +61,7 @@ void CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) [[nodiscard]] inline CDataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const size_t max_length = 4096) noexcept { - return {ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length), SER_NETWORK, INIT_PROTO_VERSION}; + return CDataStream{ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length), SER_NETWORK, INIT_PROTO_VERSION}; } [[nodiscard]] inline std::vector<std::string> ConsumeRandomLengthStringVector(FuzzedDataProvider& fuzzed_data_provider, const size_t max_vector_size = 16, const size_t max_string_length = 16) noexcept @@ -196,7 +196,7 @@ template <typename WeakEnumType, size_t size> }, [&] { WitnessUnknown witness_unknown{}; - witness_unknown.version = fuzzed_data_provider.ConsumeIntegral<int>(); + witness_unknown.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); witness_unknown.length = witness_unknown_program_1.size(); std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); @@ -269,8 +269,8 @@ inline CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcep const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); CNetAddr net_addr; if (network == Network::NET_IPV4) { - const in_addr v4_addr = { - .s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + in_addr v4_addr = {}; + v4_addr.s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); net_addr = CNetAddr{v4_addr}; } else if (network == Network::NET_IPV6) { if (fuzzed_data_provider.remaining_bytes() >= 16) { @@ -320,24 +320,9 @@ auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<N return CNode{node_id, local_services, socket, address, keyed_net_group, local_host_nonce, addr_bind, addr_name, conn_type, inbound_onion}; } } -inline std::unique_ptr<CNode> ConsumeNodeAsUniquePtr(FuzzedDataProvider& fdp, const std::optional<NodeId>& node_id_in = nullopt) { return ConsumeNode<true>(fdp, node_id_in); } +inline std::unique_ptr<CNode> ConsumeNodeAsUniquePtr(FuzzedDataProvider& fdp, const std::optional<NodeId>& node_id_in = std::nullopt) { return ConsumeNode<true>(fdp, node_id_in); } -inline void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, const std::optional<int32_t>& version_in = std::nullopt) noexcept -{ - const ServiceFlags remote_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); - const NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); - const int32_t version = version_in.value_or(fuzzed_data_provider.ConsumeIntegral<int32_t>()); - const bool filter_txs = fuzzed_data_provider.ConsumeBool(); - - node.nServices = remote_services; - node.m_permissionFlags = permission_flags; - node.nVersion = version; - node.SetCommonVersion(version); - if (node.m_tx_relay != nullptr) { - LOCK(node.m_tx_relay->cs_filter); - node.m_tx_relay->fRelayTxes = filter_txs; - } -} +void FillNode(FuzzedDataProvider& fuzzed_data_provider, CNode& node, bool init_version) noexcept; template <class T = const BasicTestingSetup> std::unique_ptr<T> MakeFuzzingContext(const std::string& chain_name = CBaseChainParams::REGTEST, const std::vector<const char*>& extra_args = {}) diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 2e5a7549b7..d57c000b92 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -7,9 +7,11 @@ #include <util/time.h> #include <boost/test/unit_test.hpp> -#include <boost/thread/thread.hpp> +#include <functional> #include <mutex> +#include <thread> +#include <vector> BOOST_AUTO_TEST_SUITE(scheduler_tests) @@ -68,16 +70,16 @@ BOOST_AUTO_TEST_CASE(manythreads) BOOST_CHECK(last > now); // As soon as these are created they will start running and servicing the queue - boost::thread_group microThreads; + std::vector<std::thread> microThreads; for (int i = 0; i < 5; i++) - microThreads.create_thread(std::bind(&CScheduler::serviceQueue, µTasks)); + microThreads.emplace_back(std::bind(&CScheduler::serviceQueue, µTasks)); UninterruptibleSleep(std::chrono::microseconds{600}); now = std::chrono::system_clock::now(); // More threads and more tasks: for (int i = 0; i < 5; i++) - microThreads.create_thread(std::bind(&CScheduler::serviceQueue, µTasks)); + microThreads.emplace_back(std::bind(&CScheduler::serviceQueue, µTasks)); for (int i = 0; i < 100; i++) { std::chrono::system_clock::time_point t = now + std::chrono::microseconds(randomMsec(rng)); std::chrono::system_clock::time_point tReschedule = now + std::chrono::microseconds(500 + randomMsec(rng)); @@ -90,7 +92,10 @@ BOOST_AUTO_TEST_CASE(manythreads) // Drain the task queue then exit threads microTasks.StopWhenDrained(); - microThreads.join_all(); // ... wait until all the threads are done + // wait until all the threads are done + for (auto& thread: microThreads) { + if (thread.joinable()) thread.join(); + } int counterSum = 0; for (int i = 0; i < 10; i++) { @@ -130,9 +135,9 @@ BOOST_AUTO_TEST_CASE(singlethreadedscheduler_ordered) // if the queues only permit execution of one task at once then // the extra threads should effectively be doing nothing // if they don't we'll get out of order behaviour - boost::thread_group threads; + std::vector<std::thread> threads; for (int i = 0; i < 5; ++i) { - threads.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); + threads.emplace_back(std::bind(&CScheduler::serviceQueue, &scheduler)); } // these are not atomic, if SinglethreadedSchedulerClient prevents @@ -156,7 +161,9 @@ BOOST_AUTO_TEST_CASE(singlethreadedscheduler_ordered) // finish up scheduler.StopWhenDrained(); - threads.join_all(); + for (auto& thread: threads) { + if (thread.joinable()) thread.join(); + } BOOST_CHECK_EQUAL(counter1, 100); BOOST_CHECK_EQUAL(counter2, 100); diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index f625b67c2a..f77cda7ba2 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -320,7 +320,7 @@ BOOST_AUTO_TEST_CASE(insert_delete) ss.insert(ss.end(), c); BOOST_CHECK_EQUAL(ss.size(), 6U); - BOOST_CHECK_EQUAL(ss[4], (char)0xff); + BOOST_CHECK_EQUAL(ss[4], 0xff); BOOST_CHECK_EQUAL(ss[5], c); ss.insert(ss.begin()+2, c); @@ -334,19 +334,14 @@ BOOST_AUTO_TEST_CASE(insert_delete) ss.erase(ss.begin()+ss.size()-1); BOOST_CHECK_EQUAL(ss.size(), 5U); - BOOST_CHECK_EQUAL(ss[4], (char)0xff); + BOOST_CHECK_EQUAL(ss[4], 0xff); ss.erase(ss.begin()+1); BOOST_CHECK_EQUAL(ss.size(), 4U); BOOST_CHECK_EQUAL(ss[0], 0); BOOST_CHECK_EQUAL(ss[1], 1); BOOST_CHECK_EQUAL(ss[2], 2); - BOOST_CHECK_EQUAL(ss[3], (char)0xff); - - // Make sure GetAndClear does the right thing: - CSerializeData d; - ss.GetAndClear(d); - BOOST_CHECK_EQUAL(ss.size(), 0U); + BOOST_CHECK_EQUAL(ss[3], 0xff); } BOOST_AUTO_TEST_CASE(class_methods) diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 8f5ee1e8e2..3079c9ff29 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE(bitstream_reader_writer) BOOST_AUTO_TEST_CASE(streams_serializedata_xor) { - std::vector<char> in; + std::vector<uint8_t> in; std::vector<char> expected_xor; std::vector<unsigned char> key; CDataStream ds(in, 0, 0); diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index 71275f69d9..3e4d1dac9e 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -6,7 +6,6 @@ #include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> -#include <boost/thread/mutex.hpp> #include <mutex> @@ -110,11 +109,6 @@ BOOST_AUTO_TEST_CASE(double_lock_mutex) TestDoubleLock<Mutex>(true /* should throw */); } -BOOST_AUTO_TEST_CASE(double_lock_boost_mutex) -{ - TestDoubleLock<boost::mutex>(true /* should throw */); -} - BOOST_AUTO_TEST_CASE(double_lock_recursive_mutex) { TestDoubleLock<RecursiveMutex>(false /* should not throw */); diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 5b35ed6976..1eb30c68d6 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -24,6 +24,7 @@ #include <util/strencodings.h> #include <validation.h> +#include <functional> #include <map> #include <string> @@ -428,12 +429,10 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) // check all inputs concurrently, with the cache PrecomputedTransactionData txdata(tx); - boost::thread_group threadGroup; CCheckQueue<CScriptCheck> scriptcheckqueue(128); CCheckQueueControl<CScriptCheck> control(&scriptcheckqueue); - for (int i=0; i<20; i++) - threadGroup.create_thread(std::bind(&CCheckQueue<CScriptCheck>::Thread, std::ref(scriptcheckqueue))); + scriptcheckqueue.StartWorkerThreads(20); std::vector<Coin> coins; for(uint32_t i = 0; i < mtx.vin.size(); i++) { @@ -455,9 +454,7 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) bool controlCheck = control.Wait(); assert(controlCheck); - - threadGroup.interrupt_all(); - threadGroup.join_all(); + scriptcheckqueue.StopWorkerThreads(); } SignatureData CombineSignatures(const CMutableTransaction& input1, const CMutableTransaction& input2, const CTransactionRef tx) diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 738f414cd0..b9f3f8c955 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -131,7 +131,7 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve // We have to run a scheduler thread to prevent ActivateBestChain // from blocking due to queue overrun. m_node.scheduler = MakeUnique<CScheduler>(); - threadGroup.create_thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); }); + m_node.scheduler->m_service_thread = std::thread([&] { TraceThread("scheduler", [&] { m_node.scheduler->serviceQueue(); }); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); @@ -143,17 +143,14 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve // Start script-checking threads. Set g_parallel_script_checks to true so they are used. constexpr int script_check_threads = 2; - for (int i = 0; i < script_check_threads; ++i) { - threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); - } + StartScriptCheckWorkerThreads(script_check_threads); g_parallel_script_checks = true; } ChainTestingSetup::~ChainTestingSetup() { if (m_node.scheduler) m_node.scheduler->stop(); - threadGroup.interrupt_all(); - threadGroup.join_all(); + StopScriptCheckWorkerThreads(); GetMainSignals().FlushBackgroundCallbacks(); GetMainSignals().UnregisterBackgroundSignalScheduler(); m_node.connman.reset(); @@ -186,7 +183,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { + if (!::ChainstateActive().ActivateBestChain(state, chainparams)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index dff8cf825e..331c1235cb 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -17,8 +17,7 @@ #include <util/string.h> #include <type_traits> - -#include <boost/thread/thread.hpp> +#include <vector> /** This is connected to the logger. Can be used to redirect logs to any other log */ extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; @@ -88,7 +87,6 @@ private: * initialization behaviour. */ struct ChainTestingSetup : public BasicTestingSetup { - boost::thread_group threadGroup; explicit ChainTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~ChainTestingSetup(); diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index ea17cb50f1..ec45d9a434 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -95,8 +95,8 @@ std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) std::shared_ptr<CBlock> MinerTestingSetup::FinalizeBlock(std::shared_ptr<CBlock> pblock) { - LOCK(cs_main); // For LookupBlockIndex - GenerateCoinbaseCommitment(*pblock, LookupBlockIndex(pblock->hashPrevBlock), Params().GetConsensus()); + LOCK(cs_main); // For g_chainman.m_blockman.LookupBlockIndex + GenerateCoinbaseCommitment(*pblock, g_chainman.m_blockman.LookupBlockIndex(pblock->hashPrevBlock), Params().GetConsensus()); pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 90ee9422ba..908ad35e1b 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -15,10 +15,11 @@ #include <util/strencodings.h> #include <util/system.h> -#include <vector> #include <deque> +#include <functional> #include <set> #include <stdlib.h> +#include <vector> #include <boost/signals2/signal.hpp> #include <boost/algorithm/string/split.hpp> diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 470e665844..c370f9e981 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -626,7 +626,7 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const uint64_t innerUsage = 0; CCoinsViewCache mempoolDuplicate(const_cast<CCoinsViewCache*>(pcoins)); - const int64_t spendheight = GetSpendHeight(mempoolDuplicate); + const int64_t spendheight = g_chainman.m_blockman.GetSpendHeight(mempoolDuplicate); std::list<const CTxMemPoolEntry*> waitingOnDependants; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { diff --git a/src/util/macros.h b/src/util/macros.h index 36ea87c0fe..0887c80fd7 100644 --- a/src/util/macros.h +++ b/src/util/macros.h @@ -8,4 +8,11 @@ #define PASTE(x, y) x ## y #define PASTE2(x, y) PASTE(x, y) +/** + * Converts the parameter X to a string after macro replacement on X has been performed. + * Don't merge these into one macro! + */ +#define STRINGIZE(X) DO_STRINGIZE(X) +#define DO_STRINGIZE(X) #X + #endif // BITCOIN_UTIL_MACROS_H diff --git a/src/util/system.cpp b/src/util/system.cpp index d1fb921642..9fe9d67411 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -3,7 +3,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <sync.h> #include <util/system.h> #ifdef HAVE_BOOST_PROCESS @@ -11,6 +10,8 @@ #endif // HAVE_BOOST_PROCESS #include <chainparamsbase.h> +#include <sync.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> @@ -310,8 +311,22 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin key[0] = '-'; #endif - if (key[0] != '-') + if (key[0] != '-') { + if (!m_accept_any_command && m_command.empty()) { + // The first non-dash arg is a registered command + Optional<unsigned int> flags = GetArgFlags(key); + if (!flags || !(*flags & ArgsManager::COMMAND)) { + error = strprintf("Invalid command '%s'", argv[i]); + return false; + } + } + m_command.push_back(key); + while (++i < argc) { + // The remaining args are command args + m_command.push_back(argv[i]); + } break; + } // Transform --foo to -foo if (key.length() > 1 && key[1] == '-') @@ -359,6 +374,26 @@ Optional<unsigned int> ArgsManager::GetArgFlags(const std::string& name) const return nullopt; } +std::optional<const ArgsManager::Command> ArgsManager::GetCommand() const +{ + Command ret; + LOCK(cs_args); + auto it = m_command.begin(); + if (it == m_command.end()) { + // No command was passed + return std::nullopt; + } + if (!m_accept_any_command) { + // The registered command + ret.command = *(it++); + } + while (it != m_command.end()) { + // The unregistered command and args (if any) + ret.args.push_back(*(it++)); + } + return ret; +} + std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) const { std::vector<std::string> result; @@ -504,8 +539,22 @@ void ArgsManager::ForceSetArg(const std::string& strArg, const std::string& strV m_settings.forced_settings[SettingName(strArg)] = strValue; } +void ArgsManager::AddCommand(const std::string& cmd, const std::string& help, const OptionsCategory& cat) +{ + Assert(cmd.find('=') == std::string::npos); + Assert(cmd.at(0) != '-'); + + LOCK(cs_args); + m_accept_any_command = false; // latch to false + std::map<std::string, Arg>& arg_map = m_available_args[cat]; + auto ret = arg_map.emplace(cmd, Arg{"", help, ArgsManager::COMMAND}); + Assert(ret.second); // Fail on duplicate commands +} + void ArgsManager::AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat) { + Assert((flags & ArgsManager::COMMAND) == 0); // use AddCommand + // Split arg name from its help param size_t eq_index = name.find('='); if (eq_index == std::string::npos) { diff --git a/src/util/system.h b/src/util/system.h index 010fc5b49f..5959bc4196 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -35,8 +35,6 @@ #include <utility> #include <vector> -#include <boost/thread/condition_variable.hpp> // for boost::thread_interrupted - class UniValue; // Application startup time (used for uptime calculation) @@ -168,7 +166,7 @@ struct SectionInfo class ArgsManager { public: - enum Flags { + enum Flags : uint32_t { // Boolean options can accept negation syntax -noOPTION or -noOPTION=1 ALLOW_BOOL = 0x01, ALLOW_INT = 0x02, @@ -183,6 +181,7 @@ public: NETWORK_ONLY = 0x200, // This argument's value is sensitive (such as a password). SENSITIVE = 0x400, + COMMAND = 0x800, }; protected: @@ -195,9 +194,11 @@ protected: mutable RecursiveMutex cs_args; util::Settings m_settings GUARDED_BY(cs_args); + std::vector<std::string> m_command GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); + bool m_accept_any_command GUARDED_BY(cs_args){true}; std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); [[nodiscard]] bool ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys = false); @@ -248,6 +249,20 @@ public: */ const std::list<SectionInfo> GetUnrecognizedSections() const; + struct Command { + /** The command (if one has been registered with AddCommand), or empty */ + std::string command; + /** + * If command is non-empty: Any args that followed it + * If command is empty: The unregistered command and any args that followed it + */ + std::vector<std::string> args; + }; + /** + * Get the command and command args (returns std::nullopt if no command provided) + */ + std::optional<const Command> GetCommand() const; + /** * Return a vector of strings of the given argument * @@ -334,6 +349,11 @@ public: void AddArg(const std::string& name, const std::string& help, unsigned int flags, const OptionsCategory& cat); /** + * Add subcommand + */ + void AddCommand(const std::string& cmd, const std::string& help, const OptionsCategory& cat); + + /** * Add many hidden arguments */ void AddHiddenArgs(const std::vector<std::string>& args); @@ -450,11 +470,6 @@ template <typename Callable> void TraceThread(const char* name, Callable func) func(); LogPrintf("%s thread exit\n", name); } - catch (const boost::thread_interrupted&) - { - LogPrintf("%s thread interrupt\n", name); - throw; - } catch (const std::exception& e) { PrintExceptionContinue(&e, name); throw; diff --git a/src/validation.cpp b/src/validation.cpp index a22c75a686..8f7d36bfd3 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -167,17 +167,19 @@ namespace { std::set<int> setDirtyFileInfo; } // anon namespace -CBlockIndex* LookupBlockIndex(const uint256& hash) +CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); - BlockMap::const_iterator it = g_chainman.BlockIndex().find(hash); - return it == g_chainman.BlockIndex().end() ? nullptr : it->second; + assert(std::addressof(g_chainman.BlockIndex()) == std::addressof(m_block_index)); + BlockMap::const_iterator it = m_block_index.find(hash); + return it == m_block_index.end() ? nullptr : it->second; } -CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) +CBlockIndex* BlockManager::FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) { AssertLockHeld(cs_main); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(*this)); // Find the latest block common to locator and chain - we expect that // locator.vHave is sorted descending by height. for (const uint256& hash : locator.vHave) { @@ -682,7 +684,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-BIP68-final"); CAmount nFees = 0; - if (!Consensus::CheckTxInputs(tx, state, m_view, GetSpendHeight(m_view), nFees)) { + if (!Consensus::CheckTxInputs(tx, state, m_view, g_chainman.m_blockman.GetSpendHeight(m_view), nFees)) { return false; // state filled in by CheckTxInputs } @@ -1262,8 +1264,8 @@ void CoinsViews::InitCache() } CChainState::CChainState(CTxMemPool& mempool, BlockManager& blockman, uint256 from_snapshot_blockhash) - : m_blockman(blockman), - m_mempool(mempool), + : m_mempool(mempool), + m_blockman(blockman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( @@ -1410,9 +1412,10 @@ bool CScriptCheck::operator()() { return VerifyScript(scriptSig, m_tx_out.scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, m_tx_out.nValue, cacheStore, *txdata), &error); } -int GetSpendHeight(const CCoinsViewCache& inputs) +int BlockManager::GetSpendHeight(const CCoinsViewCache& inputs) { - LOCK(cs_main); + AssertLockHeld(cs_main); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(*this)); CBlockIndex* pindexPrev = LookupBlockIndex(inputs.GetBestBlock()); return pindexPrev->nHeight + 1; } @@ -1763,9 +1766,14 @@ static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationSt static CCheckQueue<CScriptCheck> scriptcheckqueue(128); -void ThreadScriptCheck(int worker_num) { - util::ThreadRename(strprintf("scriptch.%i", worker_num)); - scriptcheckqueue.Thread(); +void StartScriptCheckWorkerThreads(int threads_num) +{ + scriptcheckqueue.StartWorkerThreads(threads_num); +} + +void StopScriptCheckWorkerThreads() +{ + scriptcheckqueue.StopWorkerThreads(); } VersionBitsCache versionbitscache GUARDED_BY(cs_main); @@ -2762,7 +2770,7 @@ static SynchronizationState GetSynchronizationState(bool init) return SynchronizationState::INIT_DOWNLOAD; } -static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { +static bool NotifyHeaderTip(CChainState& chainstate) LOCKS_EXCLUDED(cs_main) { bool fNotify = false; bool fInitialBlockDownload = false; static CBlockIndex* pindexHeaderOld = nullptr; @@ -2773,7 +2781,8 @@ static bool NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { if (pindexHeader != pindexHeaderOld) { fNotify = true; - fInitialBlockDownload = ::ChainstateActive().IsInitialBlockDownload(); + assert(std::addressof(::ChainstateActive()) == std::addressof(chainstate)); + fInitialBlockDownload = chainstate.IsInitialBlockDownload(); pindexHeaderOld = pindexHeader; } } @@ -2890,10 +2899,6 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar return true; } -bool ActivateBestChain(BlockValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { - return ::ChainstateActive().ActivateBestChain(state, chainparams, std::move(pblock)); -} - bool CChainState::PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) { { @@ -3393,14 +3398,14 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc return commitment; } -//! Returns last CBlockIndex* that is a checkpoint -static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) { const MapCheckpoints& checkpoints = data.mapCheckpoints; for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) { const uint256& hash = i.second; + assert(std::addressof(g_chainman.m_blockman) == std::addressof(*this)); CBlockIndex* pindex = LookupBlockIndex(hash); if (pindex) { return pindex; @@ -3418,7 +3423,7 @@ static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOC * in ConnectBlock(). * Note that -reindex-chainstate skips the validation that happens here! */ -static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, BlockManager& blockman, const CChainParams& params, const CBlockIndex* pindexPrev, int64_t nAdjustedTime) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { assert(pindexPrev != nullptr); const int nHeight = pindexPrev->nHeight + 1; @@ -3433,7 +3438,8 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // BlockIndex(). - CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(blockman)); + CBlockIndex* pcheckpoint = blockman.GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); return state.Invalid(BlockValidationResult::BLOCK_CHECKPOINT, "bad-fork-prior-to-checkpoint"); @@ -3553,11 +3559,10 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS // Check for duplicate uint256 hash = block.GetHash(); BlockMap::iterator miSelf = m_block_index.find(hash); - CBlockIndex *pindex = nullptr; if (hash != chainparams.GetConsensus().hashGenesisBlock) { if (miSelf != m_block_index.end()) { // Block header is already known. - pindex = miSelf->second; + CBlockIndex* pindex = miSelf->second; if (ppindex) *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) { @@ -3584,7 +3589,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS LogPrintf("ERROR: %s: prev block invalid\n", __func__); return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } - if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) + if (!ContextualCheckBlockHeader(block, state, *this, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), state.ToString()); /* Determine if this block descends from any block which has been found @@ -3626,8 +3631,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS } } } - if (pindex == nullptr) - pindex = AddToBlockIndex(block); + CBlockIndex* pindex = AddToBlockIndex(block); if (ppindex) *ppindex = pindex; @@ -3638,6 +3642,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS // Exposed wrapper for AcceptBlockHeader bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex) { + assert(std::addressof(::ChainstateActive()) == std::addressof(ActiveChainstate())); AssertLockNotHeld(cs_main); { LOCK(cs_main); @@ -3645,7 +3650,7 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast bool accepted = m_blockman.AcceptBlockHeader( header, state, chainparams, &pindex); - ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); + ActiveChainstate().CheckBlockIndex(chainparams.GetConsensus()); if (!accepted) { return false; @@ -3655,8 +3660,8 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& } } } - if (NotifyHeaderTip()) { - if (::ChainstateActive().IsInitialBlockDownload() && ppindex && *ppindex) { + if (NotifyHeaderTip(ActiveChainstate())) { + if (ActiveChainstate().IsInitialBlockDownload() && ppindex && *ppindex) { LogPrintf("Synchronizing blockheaders, height: %d (~%.2f%%)\n", (*ppindex)->nHeight, 100.0/((*ppindex)->nHeight+(GetAdjustedTime() - (*ppindex)->GetBlockTime()) / Params().GetConsensus().nPowTargetSpacing) * (*ppindex)->nHeight); } } @@ -3768,6 +3773,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, Block bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<const CBlock> pblock, bool fForceProcessing, bool* fNewBlock) { AssertLockNotHeld(cs_main); + assert(std::addressof(::ChainstateActive()) == std::addressof(ActiveChainstate())); { CBlockIndex *pindex = nullptr; @@ -3783,7 +3789,7 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s bool ret = CheckBlock(*pblock, state, chainparams.GetConsensus()); if (ret) { // Store to disk - ret = ::ChainstateActive().AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); + ret = ActiveChainstate().AcceptBlock(pblock, state, chainparams, &pindex, fForceProcessing, nullptr, fNewBlock); } if (!ret) { GetMainSignals().BlockChecked(*pblock, state); @@ -3791,20 +3797,27 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s } } - NotifyHeaderTip(); + NotifyHeaderTip(ActiveChainstate()); BlockValidationState state; // Only used to report errors, not invalidity - ignore it - if (!::ChainstateActive().ActivateBestChain(state, chainparams, pblock)) + if (!ActiveChainstate().ActivateBestChain(state, chainparams, pblock)) return error("%s: ActivateBestChain failed (%s)", __func__, state.ToString()); return true; } -bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) +bool TestBlockValidity(BlockValidationState& state, + const CChainParams& chainparams, + CChainState& chainstate, + const CBlock& block, + CBlockIndex* pindexPrev, + bool fCheckPOW, + bool fCheckMerkleRoot) { AssertLockHeld(cs_main); - assert(pindexPrev && pindexPrev == ::ChainActive().Tip()); - CCoinsViewCache viewNew(&::ChainstateActive().CoinsTip()); + assert(std::addressof(::ChainstateActive()) == std::addressof(chainstate)); + assert(pindexPrev && pindexPrev == chainstate.m_chain.Tip()); + CCoinsViewCache viewNew(&chainstate.CoinsTip()); uint256 block_hash(block.GetHash()); CBlockIndex indexDummy(block); indexDummy.pprev = pindexPrev; @@ -3812,13 +3825,14 @@ bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainpar indexDummy.phashBlock = &block_hash; // NOTE: CheckBlockHeader is called by CheckBlock - if (!ContextualCheckBlockHeader(block, state, chainparams, pindexPrev, GetAdjustedTime())) + assert(std::addressof(g_chainman.m_blockman) == std::addressof(chainstate.m_blockman)); + if (!ContextualCheckBlockHeader(block, state, chainstate.m_blockman, chainparams, pindexPrev, GetAdjustedTime())) return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, state.ToString()); if (!CheckBlock(block, state, chainparams.GetConsensus(), fCheckPOW, fCheckMerkleRoot)) return error("%s: Consensus::CheckBlock: %s", __func__, state.ToString()); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, state.ToString()); - if (!::ChainstateActive().ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) + if (!chainstate.ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) return false; assert(state.IsValid()); @@ -4164,7 +4178,7 @@ bool CChainState::LoadChainTip(const CChainParams& chainparams) } // Load pointer to end of best chain - CBlockIndex* pindex = LookupBlockIndex(coins_cache.GetBestBlock()); + CBlockIndex* pindex = m_blockman.LookupBlockIndex(coins_cache.GetBestBlock()); if (!pindex) { return false; } @@ -4594,7 +4608,7 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return ::ChainstateActive().LoadGenesisBlock(chainparams); } -void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp) +void CChainState::LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; @@ -4643,7 +4657,8 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi { LOCK(cs_main); // detect out of order blocks, and store them for later - if (hash != chainparams.GetConsensus().hashGenesisBlock && !LookupBlockIndex(block.hashPrevBlock)) { + assert(std::addressof(g_chainman.m_blockman) == std::addressof(m_blockman)); + if (hash != chainparams.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(block.hashPrevBlock)) { LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), block.hashPrevBlock.ToString()); if (dbp) @@ -4652,10 +4667,12 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi } // process in case the block isn't known yet - CBlockIndex* pindex = LookupBlockIndex(hash); + assert(std::addressof(g_chainman.m_blockman) == std::addressof(m_blockman)); + CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { BlockValidationState state; - if (::ChainstateActive().AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) { + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + if (AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) { nLoaded++; } if (state.IsError()) { @@ -4669,12 +4686,14 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // Activate the genesis block so normal node progress can continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { BlockValidationState state; + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); if (!ActivateBestChain(state, chainparams, nullptr)) { break; } } - NotifyHeaderTip(); + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + NotifyHeaderTip(*this); // Recursively process earlier encountered successors of this block std::deque<uint256> queue; @@ -4692,7 +4711,8 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi head.ToString()); LOCK(cs_main); BlockValidationState dummy; - if (::ChainstateActive().AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + if (AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) { nLoaded++; queue.push_back(pblockrecursive->GetHash()); @@ -4700,7 +4720,8 @@ void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi } range.first++; mapBlocksUnknownParent.erase(it); - NotifyHeaderTip(); + assert(std::addressof(::ChainstateActive()) == std::addressof(*this)); + NotifyHeaderTip(*this); } } } catch (const std::exception& e) { @@ -5138,6 +5159,7 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin } Optional<uint256> ChainstateManager::SnapshotBlockhash() const { + LOCK(::cs_main); if (m_active_chainstate != nullptr) { // If a snapshot chainstate exists, it will always be our active. return m_active_chainstate->m_from_snapshot_blockhash; @@ -5147,6 +5169,7 @@ Optional<uint256> ChainstateManager::SnapshotBlockhash() const { std::vector<CChainState*> ChainstateManager::GetAll() { + LOCK(::cs_main); std::vector<CChainState*> out; if (!IsSnapshotValidated() && m_ibd_chainstate) { @@ -5184,17 +5207,20 @@ CChainState& ChainstateManager::InitializeChainstate(CTxMemPool& mempool, const CChainState& ChainstateManager::ActiveChainstate() const { + LOCK(::cs_main); assert(m_active_chainstate); return *m_active_chainstate; } bool ChainstateManager::IsSnapshotActive() const { + LOCK(::cs_main); return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); } CChainState& ChainstateManager::ValidatedChainstate() const { + LOCK(::cs_main); if (m_snapshot_chainstate && IsSnapshotValidated()) { return *m_snapshot_chainstate.get(); } @@ -5204,6 +5230,7 @@ CChainState& ChainstateManager::ValidatedChainstate() const bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const { + LOCK(::cs_main); return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); } @@ -5219,6 +5246,7 @@ void ChainstateManager::Unload() void ChainstateManager::Reset() { + LOCK(::cs_main); m_ibd_chainstate.reset(); m_snapshot_chainstate.reset(); m_active_chainstate = nullptr; diff --git a/src/validation.h b/src/validation.h index 1b4e40cd53..e85c7bbf1a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -40,6 +40,7 @@ class CBlockIndex; class CBlockTreeDB; class CBlockUndo; class CChainParams; +struct CCheckpointData; class CInv; class CConnman; class CScriptCheck; @@ -143,14 +144,14 @@ extern const std::vector<std::string> CHECKLEVEL_DOC; FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ fs::path GetBlockPosFilename(const FlatFilePos &pos); -/** Import blocks from an external file */ -void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); /** Unload database information */ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman); -/** Run an instance of the script checking thread */ -void ThreadScriptCheck(int worker_num); +/** Run instances of script checking worker threads */ +void StartScriptCheckWorkerThreads(int threads_num); +/** Stop all of the script checking worker threads */ +void StopScriptCheckWorkerThreads(); /** * Return transaction from the block at block_index. * If block_index is not provided, fall back to mempool. @@ -164,13 +165,6 @@ void ThreadScriptCheck(int worker_num); * @returns The tx if found, otherwise nullptr */ CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); -/** - * Find the best known block, and make it the tip of the block chain - * - * May not be called with cs_main held. May not be called in a - * validationinterface callback. - */ -bool ActivateBestChain(BlockValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ @@ -289,7 +283,13 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true, bool fCheckMerkleRoot = true); /** Check a block is completely valid from start to finish (only works on top of our current best block) */ -bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW = true, bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool TestBlockValidity(BlockValidationState& state, + const CChainParams& chainparams, + CChainState& chainstate, + const CBlock& block, + CBlockIndex* pindexPrev, + bool fCheckPOW = true, + bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Check whether witness commitments are required for a block, and whether to enforce NULLDUMMY (BIP 147) rules. * Note that transaction witness validation rules are always enforced when P2SH is enforced. */ @@ -309,11 +309,6 @@ public: bool VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth); }; -CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - -/** Find the last common block between the parameter chain and a locator. */ -CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - enum DisconnectResult { DISCONNECT_OK, // All good. @@ -431,6 +426,21 @@ public: const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + CBlockIndex* LookupBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** Find the last common block between the parameter chain and a locator. */ + CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Returns last CBlockIndex* that is a checkpoint + CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** + * Return the spend height, which is one more than the inputs.GetBestBlock(). + * While checking, GetBestBlock() refers to the parent block. (protected by cs_main) + * This is also true for mempool checks. + */ + int GetSpendHeight(const CCoinsViewCache& inputs) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + ~BlockManager() { Unload(); } @@ -523,11 +533,6 @@ protected: */ mutable std::atomic<bool> m_cached_finished_ibd{false}; - //! Reference to a BlockManager instance which itself is shared across all - //! CChainState instances. Keeping a local reference allows us to test more - //! easily as opposed to referencing a global. - BlockManager& m_blockman; - //! mempool that is kept in sync with the chain CTxMemPool& m_mempool; @@ -535,6 +540,10 @@ protected: std::unique_ptr<CoinsViews> m_coins_views; public: + //! Reference to a BlockManager instance which itself is shared across all + //! CChainState instances. + BlockManager& m_blockman; + explicit CChainState(CTxMemPool& mempool, BlockManager& blockman, uint256 from_snapshot_blockhash = uint256()); /** @@ -611,6 +620,9 @@ public: bool ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** Import blocks from an external file */ + void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr); + /** * Update the on-disk chain state. * The caches and indexes are flushed depending on the mode we're called with @@ -636,9 +648,10 @@ public: void PruneAndFlush(); /** - * Make the best chain active, in multiple steps. The result is either failure - * or an activated best chain. pblock is either nullptr or a pointer to a block - * that is already loaded (to avoid loading it again from disk). + * Find the best known block, and make it the tip of the block chain. The + * result is either failure or an activated best chain. pblock is either + * nullptr or a pointer to a block that is already loaded (to avoid loading + * it again from disk). * * ActivateBestChain is split into steps (see ActivateBestChainStep) so that * we avoid holding cs_main for an extended period of time; the length of this @@ -652,7 +665,7 @@ public: bool ActivateBestChain( BlockValidationState& state, const CChainParams& chainparams, - std::shared_ptr<const CBlock> pblock) LOCKS_EXCLUDED(cs_main); + std::shared_ptr<const CBlock> pblock = nullptr) LOCKS_EXCLUDED(cs_main); bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -789,7 +802,7 @@ private: //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - std::unique_ptr<CChainState> m_ibd_chainstate; + std::unique_ptr<CChainState> m_ibd_chainstate GUARDED_BY(::cs_main); //! A chainstate initialized on the basis of a UTXO snapshot. If this is //! non-null, it is always our active chainstate. @@ -802,7 +815,7 @@ private: //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - std::unique_ptr<CChainState> m_snapshot_chainstate; + std::unique_ptr<CChainState> m_snapshot_chainstate GUARDED_BY(::cs_main); //! Points to either the ibd or snapshot chainstate; indicates our //! most-work chain. @@ -815,7 +828,7 @@ private: //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into //! that call. - CChainState* m_active_chainstate{nullptr}; + CChainState* m_active_chainstate GUARDED_BY(::cs_main) {nullptr}; //! If true, the assumed-valid chainstate has been fully validated //! by the background validation chainstate. @@ -943,13 +956,6 @@ CChain& ChainActive(); /** Global variable that points to the active block tree (protected by cs_main) */ extern std::unique_ptr<CBlockTreeDB> pblocktree; -/** - * Return the spend height, which is one more than the inputs.GetBestBlock(). - * While checking, GetBestBlock() refers to the parent block. (protected by cs_main) - * This is also true for mempool checks. - */ -int GetSpendHeight(const CCoinsViewCache& inputs); - extern VersionBitsCache versionbitscache; /** diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index c0d107bf39..ad40e6da9a 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -488,9 +488,9 @@ bool BerkeleyDatabase::Rewrite(const char* pszSkip) break; } if (pszSkip && - strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) + strncmp((const char*)ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) continue; - if (strncmp(ssKey.data(), "\x07version", 8) == 0) { + if (strncmp((const char*)ssKey.data(), "\x07version", 8) == 0) { // Update version: ssValue.clear(); ssValue << CLIENT_VERSION; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index a2dea84d17..10f89e3a6f 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -300,8 +300,26 @@ bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& group ******************************************************************************/ -void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants) { +void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants, bool positive_only) { + // Compute the effective value first + const CAmount coin_fee = output.m_input_bytes < 0 ? 0 : m_effective_feerate.GetFee(output.m_input_bytes); + const CAmount ev = output.txout.nValue - coin_fee; + + // Filter for positive only here before adding the coin + if (positive_only && ev <= 0) return; + m_outputs.push_back(output); + CInputCoin& coin = m_outputs.back(); + + coin.m_fee = coin_fee; + fee += coin.m_fee; + + coin.m_long_term_fee = coin.m_input_bytes < 0 ? 0 : m_long_term_feerate.GetFee(coin.m_input_bytes); + long_term_fee += coin.m_long_term_fee; + + coin.effective_value = ev; + effective_value += coin.effective_value; + m_from_me &= from_me; m_value += output.txout.nValue; m_depth = std::min(m_depth, depth); @@ -312,20 +330,6 @@ void OutputGroup::Insert(const CInputCoin& output, int depth, bool from_me, size // descendants is the count as seen from the top ancestor, not the descendants as seen from the // coin itself; thus, this value is counted as the max, not the sum m_descendants = std::max(m_descendants, descendants); - effective_value += output.effective_value; - fee += output.m_fee; - long_term_fee += output.m_long_term_fee; -} - -std::vector<CInputCoin>::iterator OutputGroup::Discard(const CInputCoin& output) { - auto it = m_outputs.begin(); - while (it != m_outputs.end() && it->outpoint != output.outpoint) ++it; - if (it == m_outputs.end()) return it; - m_value -= output.txout.nValue; - effective_value -= output.effective_value; - fee -= output.m_fee; - long_term_fee -= output.m_long_term_fee; - return m_outputs.erase(it); } bool OutputGroup::EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const @@ -334,35 +338,3 @@ bool OutputGroup::EligibleForSpending(const CoinEligibilityFilter& eligibility_f && m_ancestors <= eligibility_filter.max_ancestors && m_descendants <= eligibility_filter.max_descendants; } - -void OutputGroup::SetFees(const CFeeRate effective_feerate, const CFeeRate long_term_feerate) -{ - fee = 0; - long_term_fee = 0; - effective_value = 0; - for (CInputCoin& coin : m_outputs) { - coin.m_fee = coin.m_input_bytes < 0 ? 0 : effective_feerate.GetFee(coin.m_input_bytes); - fee += coin.m_fee; - - coin.m_long_term_fee = coin.m_input_bytes < 0 ? 0 : long_term_feerate.GetFee(coin.m_input_bytes); - long_term_fee += coin.m_long_term_fee; - - coin.effective_value = coin.txout.nValue - coin.m_fee; - effective_value += coin.effective_value; - } -} - -OutputGroup OutputGroup::GetPositiveOnlyGroup() -{ - OutputGroup group(*this); - for (auto it = group.m_outputs.begin(); it != group.m_outputs.end(); ) { - const CInputCoin& coin = *it; - // Only include outputs that are positive effective value (i.e. not dust) - if (coin.effective_value <= 0) { - it = group.Discard(coin); - } else { - ++it; - } - } - return group; -} diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index cf9768b927..f0e1addaf1 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -6,11 +6,10 @@ #define BITCOIN_WALLET_COINSELECTION_H #include <amount.h> +#include <policy/feerate.h> #include <primitives/transaction.h> #include <random.h> -class CFeeRate; - //! target minimum change amount static constexpr CAmount MIN_CHANGE{COIN / 100}; //! final minimum change amount after paying for fees @@ -63,9 +62,11 @@ struct CoinEligibilityFilter const int conf_theirs; const uint64_t max_ancestors; const uint64_t max_descendants; + const bool m_include_partial_groups{false}; //! Include partial destination groups when avoid_reuse and there are full groups CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_ancestors) {} CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants) {} + CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants, bool include_partial) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants), m_include_partial_groups(include_partial) {} }; struct OutputGroup @@ -78,27 +79,18 @@ struct OutputGroup size_t m_descendants{0}; CAmount effective_value{0}; CAmount fee{0}; + CFeeRate m_effective_feerate{0}; CAmount long_term_fee{0}; + CFeeRate m_long_term_feerate{0}; OutputGroup() {} - OutputGroup(std::vector<CInputCoin>&& outputs, bool from_me, CAmount value, int depth, size_t ancestors, size_t descendants) - : m_outputs(std::move(outputs)) - , m_from_me(from_me) - , m_value(value) - , m_depth(depth) - , m_ancestors(ancestors) - , m_descendants(descendants) + OutputGroup(const CFeeRate& effective_feerate, const CFeeRate& long_term_feerate) : + m_effective_feerate(effective_feerate), + m_long_term_feerate(long_term_feerate) {} - OutputGroup(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants) : OutputGroup() { - Insert(output, depth, from_me, ancestors, descendants); - } - void Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants); - std::vector<CInputCoin>::iterator Discard(const CInputCoin& output); - bool EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const; - //! Update the OutputGroup's fee, long_term_fee, and effective_value based on the given feerates - void SetFees(const CFeeRate effective_feerate, const CFeeRate long_term_feerate); - OutputGroup GetPositiveOnlyGroup(); + void Insert(const CInputCoin& output, int depth, bool from_me, size_t ancestors, size_t descendants, bool positive_only); + bool EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const; }; bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& target_value, const CAmount& cost_of_change, std::set<CInputCoin>& out_set, CAmount& value_ret, CAmount not_input_fees); diff --git a/src/wallet/ismine.h b/src/wallet/ismine.h index 5cdd7dff80..38ed7e7770 100644 --- a/src/wallet/ismine.h +++ b/src/wallet/ismine.h @@ -14,7 +14,27 @@ class CWallet; class CScript; -/** IsMine() return codes */ +/** + * IsMine() return codes, which depend on ScriptPubKeyMan implementation. + * Not every ScriptPubKeyMan covers all types, please refer to + * https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.21.0.md#ismine-semantics + * for better understanding. + * + * For LegacyScriptPubKeyMan, + * ISMINE_NO: the scriptPubKey is not in the wallet; + * ISMINE_WATCH_ONLY: the scriptPubKey has been imported into the wallet; + * ISMINE_SPENDABLE: the scriptPubKey corresponds to an address owned by the wallet user (can spend with the private key); + * ISMINE_USED: the scriptPubKey corresponds to a used address owned by the wallet user; + * ISMINE_ALL: all ISMINE flags except for USED; + * ISMINE_ALL_USED: all ISMINE flags including USED; + * ISMINE_ENUM_ELEMENTS: the number of isminetype enum elements. + * + * For DescriptorScriptPubKeyMan and future ScriptPubKeyMan, + * ISMINE_NO: the scriptPubKey is not in the wallet; + * ISMINE_SPENDABLE: the scriptPubKey matches a scriptPubKey in the wallet. + * ISMINE_USED: the scriptPubKey corresponds to a used address owned by the wallet user. + * + */ enum isminetype : unsigned int { ISMINE_NO = 0, diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 6b46868d10..8505ddc309 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -1740,3 +1740,72 @@ RPCHelpMan importdescriptors() }, }; } + +RPCHelpMan listdescriptors() +{ + return RPCHelpMan{ + "listdescriptors", + "\nList descriptors imported into a descriptor-enabled wallet.", + {}, + RPCResult{ + RPCResult::Type::ARR, "", "Response is an array of descriptor objects", + { + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "desc", "Descriptor string representation"}, + {RPCResult::Type::NUM, "timestamp", "The creation time of the descriptor"}, + {RPCResult::Type::BOOL, "active", "Activeness flag"}, + {RPCResult::Type::BOOL, "internal", true, "Whether this is internal or external descriptor; defined only for active descriptors"}, + {RPCResult::Type::ARR_FIXED, "range", true, "Defined only for ranged descriptors", { + {RPCResult::Type::NUM, "", "Range start inclusive"}, + {RPCResult::Type::NUM, "", "Range end inclusive"}, + }}, + {RPCResult::Type::NUM, "next", true, "The next index to generate addresses from; defined only for ranged descriptors"}, + }}, + } + }, + RPCExamples{ + HelpExampleCli("listdescriptors", "") + HelpExampleRpc("listdescriptors", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + if (!wallet) return NullUniValue; + + if (!wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "listdescriptors is not available for non-descriptor wallets"); + } + + LOCK(wallet->cs_wallet); + + UniValue response(UniValue::VARR); + const auto active_spk_mans = wallet->GetActiveScriptPubKeyMans(); + for (const auto& spk_man : wallet->GetAllScriptPubKeyMans()) { + const auto desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man); + if (!desc_spk_man) { + throw JSONRPCError(RPC_WALLET_ERROR, "Unexpected ScriptPubKey manager type."); + } + UniValue spk(UniValue::VOBJ); + LOCK(desc_spk_man->cs_desc_man); + const auto& wallet_descriptor = desc_spk_man->GetWalletDescriptor(); + spk.pushKV("desc", wallet_descriptor.descriptor->ToString()); + spk.pushKV("timestamp", wallet_descriptor.creation_time); + const bool active = active_spk_mans.count(desc_spk_man) != 0; + spk.pushKV("active", active); + const auto& type = wallet_descriptor.descriptor->GetOutputType(); + if (active && type != nullopt) { + spk.pushKV("internal", wallet->GetScriptPubKeyMan(*type, true) == desc_spk_man); + } + if (wallet_descriptor.descriptor->IsRange()) { + UniValue range(UniValue::VARR); + range.push_back(wallet_descriptor.range_start); + range.push_back(wallet_descriptor.range_end - 1); + spk.pushKV("range", range); + spk.pushKV("next", wallet_descriptor.next_index); + } + response.push_back(spk); + } + + return response; +}, + }; +} diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 9db327c913..46de273d63 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2615,7 +2615,18 @@ static RPCHelpMan loadwallet() if (!wallet) { // Map bad format to not found, since bad format is returned when the // wallet directory exists, but doesn't contain a data file. - RPCErrorCode code = status == DatabaseStatus::FAILED_NOT_FOUND || status == DatabaseStatus::FAILED_BAD_FORMAT ? RPC_WALLET_NOT_FOUND : RPC_WALLET_ERROR; + RPCErrorCode code = RPC_WALLET_ERROR; + switch (status) { + case DatabaseStatus::FAILED_NOT_FOUND: + case DatabaseStatus::FAILED_BAD_FORMAT: + code = RPC_WALLET_NOT_FOUND; + break; + case DatabaseStatus::FAILED_ALREADY_LOADED: + code = RPC_WALLET_ALREADY_LOADED; + break; + default: // RPC_WALLET_ERROR is returned for all other cases. + break; + } throw JSONRPCError(code, error.original); } @@ -3820,13 +3831,19 @@ RPCHelpMan getaddressinfo() LOCK(pwallet->cs_wallet); - UniValue ret(UniValue::VOBJ); - CTxDestination dest = DecodeDestination(request.params[0].get_str()); + std::string error_msg; + CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg); + // Make sure the destination is valid if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + // Set generic error message in case 'DecodeDestination' didn't set it + if (error_msg.empty()) error_msg = "Invalid address"; + + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error_msg); } + UniValue ret(UniValue::VOBJ); + std::string currentAddress = EncodeDestination(dest); ret.pushKV("address", currentAddress); @@ -4534,73 +4551,75 @@ RPCHelpMan importprunedfunds(); RPCHelpMan removeprunedfunds(); RPCHelpMan importmulti(); RPCHelpMan importdescriptors(); +RPCHelpMan listdescriptors(); Span<const CRPCCommand> GetWalletRPCCommands() { // clang-format off static const CRPCCommand commands[] = -{ // category name actor (function) argNames - // --------------------- ------------------------ ----------------------- ---------- - { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options","iswitness"} }, - { "wallet", "abandontransaction", &abandontransaction, {"txid"} }, - { "wallet", "abortrescan", &abortrescan, {} }, - { "wallet", "addmultisigaddress", &addmultisigaddress, {"nrequired","keys","label","address_type"} }, - { "wallet", "backupwallet", &backupwallet, {"destination"} }, - { "wallet", "bumpfee", &bumpfee, {"txid", "options"} }, - { "wallet", "psbtbumpfee", &psbtbumpfee, {"txid", "options"} }, - { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse", "descriptors", "load_on_startup"} }, - { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, - { "wallet", "dumpwallet", &dumpwallet, {"filename"} }, - { "wallet", "encryptwallet", &encryptwallet, {"passphrase"} }, - { "wallet", "getaddressesbylabel", &getaddressesbylabel, {"label"} }, - { "wallet", "getaddressinfo", &getaddressinfo, {"address"} }, - { "wallet", "getbalance", &getbalance, {"dummy","minconf","include_watchonly","avoid_reuse"} }, - { "wallet", "getnewaddress", &getnewaddress, {"label","address_type"} }, - { "wallet", "getrawchangeaddress", &getrawchangeaddress, {"address_type"} }, - { "wallet", "getreceivedbyaddress", &getreceivedbyaddress, {"address","minconf"} }, - { "wallet", "getreceivedbylabel", &getreceivedbylabel, {"label","minconf"} }, - { "wallet", "gettransaction", &gettransaction, {"txid","include_watchonly","verbose"} }, - { "wallet", "getunconfirmedbalance", &getunconfirmedbalance, {} }, - { "wallet", "getbalances", &getbalances, {} }, - { "wallet", "getwalletinfo", &getwalletinfo, {} }, - { "wallet", "importaddress", &importaddress, {"address","label","rescan","p2sh"} }, - { "wallet", "importdescriptors", &importdescriptors, {"requests"} }, - { "wallet", "importmulti", &importmulti, {"requests","options"} }, - { "wallet", "importprivkey", &importprivkey, {"privkey","label","rescan"} }, - { "wallet", "importprunedfunds", &importprunedfunds, {"rawtransaction","txoutproof"} }, - { "wallet", "importpubkey", &importpubkey, {"pubkey","label","rescan"} }, - { "wallet", "importwallet", &importwallet, {"filename"} }, - { "wallet", "keypoolrefill", &keypoolrefill, {"newsize"} }, - { "wallet", "listaddressgroupings", &listaddressgroupings, {} }, - { "wallet", "listlabels", &listlabels, {"purpose"} }, - { "wallet", "listlockunspent", &listlockunspent, {} }, - { "wallet", "listreceivedbyaddress", &listreceivedbyaddress, {"minconf","include_empty","include_watchonly","address_filter"} }, - { "wallet", "listreceivedbylabel", &listreceivedbylabel, {"minconf","include_empty","include_watchonly"} }, - { "wallet", "listsinceblock", &listsinceblock, {"blockhash","target_confirmations","include_watchonly","include_removed"} }, - { "wallet", "listtransactions", &listtransactions, {"label|dummy","count","skip","include_watchonly"} }, - { "wallet", "listunspent", &listunspent, {"minconf","maxconf","addresses","include_unsafe","query_options"} }, - { "wallet", "listwalletdir", &listwalletdir, {} }, - { "wallet", "listwallets", &listwallets, {} }, - { "wallet", "loadwallet", &loadwallet, {"filename", "load_on_startup"} }, - { "wallet", "lockunspent", &lockunspent, {"unlock","transactions"} }, - { "wallet", "removeprunedfunds", &removeprunedfunds, {"txid"} }, - { "wallet", "rescanblockchain", &rescanblockchain, {"start_height", "stop_height"} }, - { "wallet", "send", &send, {"outputs","conf_target","estimate_mode","fee_rate","options"} }, - { "wallet", "sendmany", &sendmany, {"dummy","amounts","minconf","comment","subtractfeefrom","replaceable","conf_target","estimate_mode","fee_rate","verbose"} }, - { "wallet", "sendtoaddress", &sendtoaddress, {"address","amount","comment","comment_to","subtractfeefromamount","replaceable","conf_target","estimate_mode","avoid_reuse","fee_rate","verbose"} }, - { "wallet", "sethdseed", &sethdseed, {"newkeypool","seed"} }, - { "wallet", "setlabel", &setlabel, {"address","label"} }, - { "wallet", "settxfee", &settxfee, {"amount"} }, - { "wallet", "setwalletflag", &setwalletflag, {"flag","value"} }, - { "wallet", "signmessage", &signmessage, {"address","message"} }, - { "wallet", "signrawtransactionwithwallet", &signrawtransactionwithwallet, {"hexstring","prevtxs","sighashtype"} }, - { "wallet", "unloadwallet", &unloadwallet, {"wallet_name", "load_on_startup"} }, - { "wallet", "upgradewallet", &upgradewallet, {"version"} }, - { "wallet", "walletcreatefundedpsbt", &walletcreatefundedpsbt, {"inputs","outputs","locktime","options","bip32derivs"} }, - { "wallet", "walletlock", &walletlock, {} }, - { "wallet", "walletpassphrase", &walletpassphrase, {"passphrase","timeout"} }, - { "wallet", "walletpassphrasechange", &walletpassphrasechange, {"oldpassphrase","newpassphrase"} }, - { "wallet", "walletprocesspsbt", &walletprocesspsbt, {"psbt","sign","sighashtype","bip32derivs"} }, +{ // category actor (function) + // ------------------ ------------------------ + { "rawtransactions", &fundrawtransaction, }, + { "wallet", &abandontransaction, }, + { "wallet", &abortrescan, }, + { "wallet", &addmultisigaddress, }, + { "wallet", &backupwallet, }, + { "wallet", &bumpfee, }, + { "wallet", &psbtbumpfee, }, + { "wallet", &createwallet, }, + { "wallet", &dumpprivkey, }, + { "wallet", &dumpwallet, }, + { "wallet", &encryptwallet, }, + { "wallet", &getaddressesbylabel, }, + { "wallet", &getaddressinfo, }, + { "wallet", &getbalance, }, + { "wallet", &getnewaddress, }, + { "wallet", &getrawchangeaddress, }, + { "wallet", &getreceivedbyaddress, }, + { "wallet", &getreceivedbylabel, }, + { "wallet", &gettransaction, }, + { "wallet", &getunconfirmedbalance, }, + { "wallet", &getbalances, }, + { "wallet", &getwalletinfo, }, + { "wallet", &importaddress, }, + { "wallet", &importdescriptors, }, + { "wallet", &importmulti, }, + { "wallet", &importprivkey, }, + { "wallet", &importprunedfunds, }, + { "wallet", &importpubkey, }, + { "wallet", &importwallet, }, + { "wallet", &keypoolrefill, }, + { "wallet", &listaddressgroupings, }, + { "wallet", &listdescriptors, }, + { "wallet", &listlabels, }, + { "wallet", &listlockunspent, }, + { "wallet", &listreceivedbyaddress, }, + { "wallet", &listreceivedbylabel, }, + { "wallet", &listsinceblock, }, + { "wallet", &listtransactions, }, + { "wallet", &listunspent, }, + { "wallet", &listwalletdir, }, + { "wallet", &listwallets, }, + { "wallet", &loadwallet, }, + { "wallet", &lockunspent, }, + { "wallet", &removeprunedfunds, }, + { "wallet", &rescanblockchain, }, + { "wallet", &send, }, + { "wallet", &sendmany, }, + { "wallet", &sendtoaddress, }, + { "wallet", &sethdseed, }, + { "wallet", &setlabel, }, + { "wallet", &settxfee, }, + { "wallet", &setwalletflag, }, + { "wallet", &signmessage, }, + { "wallet", &signrawtransactionwithwallet, }, + { "wallet", &unloadwallet, }, + { "wallet", &upgradewallet, }, + { "wallet", &walletcreatefundedpsbt, }, + { "wallet", &walletlock, }, + { "wallet", &walletpassphrase, }, + { "wallet", &walletpassphrasechange, }, + { "wallet", &walletprocesspsbt, }, }; // clang-format on return MakeSpan(commands); diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 202804c9ff..ffac78d752 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -35,7 +35,7 @@ static CAmount balance = 0; CoinEligibilityFilter filter_standard(1, 6, 0); CoinEligibilityFilter filter_confirmed(1, 1, 0); CoinEligibilityFilter filter_standard_extra(6, 6, 0); -CoinSelectionParams coin_selection_params(false, 0, 0, CFeeRate(0), 0); +CoinSelectionParams coin_selection_params(false, 0, 0, CFeeRate(0), 0, false); static void add_coin(const CAmount& nValue, int nInput, std::vector<CInputCoin>& set) { @@ -115,7 +115,10 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<CInputCoin>& coins { static std::vector<OutputGroup> static_groups; static_groups.clear(); - for (auto& coin : coins) static_groups.emplace_back(coin, 0, true, 0, 0); + for (auto& coin : coins) { + static_groups.emplace_back(); + static_groups.back().Insert(coin, 0, true, 0, 0, false); + } return static_groups; } @@ -123,7 +126,10 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) { static std::vector<OutputGroup> static_groups; static_groups.clear(); - for (auto& coin : coins) static_groups.emplace_back(coin.GetInputCoin(), coin.nDepth, coin.tx->m_amounts[CWalletTx::DEBIT].m_cached[ISMINE_SPENDABLE] && coin.tx->m_amounts[CWalletTx::DEBIT].m_value[ISMINE_SPENDABLE] == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0); + for (auto& coin : coins) { + static_groups.emplace_back(); + static_groups.back().Insert(coin.GetInputCoin(), coin.nDepth, coin.tx->m_amounts[CWalletTx::DEBIT].m_cached[ISMINE_SPENDABLE] && coin.tx->m_amounts[CWalletTx::DEBIT].m_value[ISMINE_SPENDABLE] == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0, false); + } return static_groups; } @@ -263,22 +269,22 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) } // Make sure that effective value is working in SelectCoinsMinConf when BnB is used - CoinSelectionParams coin_selection_params_bnb(true, 0, 0, CFeeRate(3000), 0); + CoinSelectionParams coin_selection_params_bnb(true, 0, 0, CFeeRate(3000), 0, false); CoinSet setCoinsRet; CAmount nValueRet; bool bnb_used; empty_wallet(); add_coin(1); vCoins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); // Test fees subtracted from output: empty_wallet(); add_coin(1 * CENT); vCoins.at(0).nInputBytes = 40; - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); coin_selection_params_bnb.m_subtract_fee_outputs = true; - BOOST_CHECK(testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); // Make sure that can use BnB when there are preset inputs @@ -317,24 +323,24 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) empty_wallet(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); add_coin(1*CENT, 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // but we can find a new 1 cent - BOOST_CHECK( testWallet.SelectCoinsMinConf( 1 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 1 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); add_coin(2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!testWallet.SelectCoinsMinConf( 3 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf( 3 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // we can make 3 cents of new coins - BOOST_CHECK( testWallet.SelectCoinsMinConf( 3 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 3 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 3 * CENT); add_coin(5*CENT); // add a mature 5 cent coin, @@ -344,33 +350,33 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard_extra, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(38 * CENT, filter_standard_extra, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // but we can make 37 cents if we accept new coins from ourself - BOOST_CHECK( testWallet.SelectCoinsMinConf(37 * CENT, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(37 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 37 * CENT); // and we can make 38 cents if we accept all new coins - BOOST_CHECK( testWallet.SelectCoinsMinConf(38 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(38 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - BOOST_CHECK( testWallet.SelectCoinsMinConf(34 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(34 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - BOOST_CHECK( testWallet.SelectCoinsMinConf( 7 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 7 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 7 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - BOOST_CHECK( testWallet.SelectCoinsMinConf( 8 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 8 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK(nValueRet == 8 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - BOOST_CHECK( testWallet.SelectCoinsMinConf( 9 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf( 9 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 10 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -384,30 +390,30 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - BOOST_CHECK( testWallet.SelectCoinsMinConf(71 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(!testWallet.SelectCoinsMinConf(72 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(71 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(!testWallet.SelectCoinsMinConf(72 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); add_coin( 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); add_coin( 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - BOOST_CHECK( testWallet.SelectCoinsMinConf(11 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(11 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 11 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -416,11 +422,11 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin( 2*COIN); add_coin( 3*COIN); add_coin( 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - BOOST_CHECK( testWallet.SelectCoinsMinConf(95 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(95 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - BOOST_CHECK( testWallet.SelectCoinsMinConf(195 * CENT, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(195 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -435,14 +441,14 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // try making 1 * MIN_CHANGE from the 1.5 * MIN_CHANGE // we'll get change smaller than MIN_CHANGE whatever happens, so can expect MIN_CHANGE exactly - BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // but if we add a bigger coin, small change is avoided add_coin(1111*MIN_CHANGE); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // if we add more small coins: @@ -450,7 +456,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 7 / 10); // and try again to make 1.0 * MIN_CHANGE - BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) @@ -459,7 +465,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int j = 0; j < 20; j++) add_coin(50000 * COIN); - BOOST_CHECK( testWallet.SelectCoinsMinConf(500000 * COIN, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(500000 * COIN, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 10U); // in ten coins @@ -472,7 +478,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 6 / 10); add_coin(MIN_CHANGE * 7 / 10); add_coin(1111 * MIN_CHANGE); - BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1111 * MIN_CHANGE); // we get the bigger coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -482,7 +488,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 6 / 10); add_coin(MIN_CHANGE * 8 / 10); add_coin(1111 * MIN_CHANGE); - BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK( testWallet.SelectCoinsMinConf(MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // in two coins 0.4+0.6 @@ -493,12 +499,12 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 100); // trying to make 100.01 from these three coins - BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 10001 / 100, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 10001 / 100, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 9990 / 100, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(MIN_CHANGE * 9990 / 100, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 101 * MIN_CHANGE); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); } @@ -512,7 +518,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. for (int i = 0; i < RUN_TESTS; i++) { - BOOST_CHECK(testWallet.SelectCoinsMinConf(2000, filter_confirmed, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(2000, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); if (amt - 2000 < MIN_CHANGE) { // needs more than one input: @@ -538,8 +544,8 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int i = 0; i < RUN_TESTS; i++) { // picking 50 from 100 coins doesn't depend on the shuffle, // but does depend on randomness in the stochastic approximation code - BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, GroupCoins(vCoins), setCoinsRet , nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, GroupCoins(vCoins), setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, vCoins, setCoinsRet , nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(50 * COIN, filter_standard, vCoins, setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK(!equal_sets(setCoinsRet, setCoinsRet2)); int fails = 0; @@ -547,8 +553,8 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) { // selecting 1 from 100 identical coins depends on the shuffle; this test will fail 1% of the time // run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, GroupCoins(vCoins), setCoinsRet , nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, GroupCoins(vCoins), setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, vCoins, setCoinsRet , nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(COIN, filter_standard, vCoins, setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } @@ -570,8 +576,8 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) { // selecting 1 from 100 identical coins depends on the shuffle; this test will fail 1% of the time // run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, GroupCoins(vCoins), setCoinsRet , nValueRet, coin_selection_params, bnb_used)); - BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, GroupCoins(vCoins), setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, vCoins, setCoinsRet , nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(90*CENT, filter_standard, vCoins, setCoinsRet2, nValueRet, coin_selection_params, bnb_used)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } @@ -598,7 +604,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) add_coin(1000 * COIN); add_coin(3 * COIN); - BOOST_CHECK(testWallet.SelectCoinsMinConf(1003 * COIN, filter_standard, GroupCoins(vCoins), setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(1003 * COIN, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); BOOST_CHECK_EQUAL(nValueRet, 1003 * COIN); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -633,13 +639,13 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) CAmount target = rand.randrange(balance - 1000) + 1000; // Perform selection - CoinSelectionParams coin_selection_params_knapsack(false, 34, 148, CFeeRate(0), 0); - CoinSelectionParams coin_selection_params_bnb(true, 34, 148, CFeeRate(0), 0); + CoinSelectionParams coin_selection_params_knapsack(false, 34, 148, CFeeRate(0), 0, false); + CoinSelectionParams coin_selection_params_bnb(true, 34, 148, CFeeRate(0), 0, false); CoinSet out_set; CAmount out_value = 0; bool bnb_used = false; - BOOST_CHECK(testWallet.SelectCoinsMinConf(target, filter_standard, GroupCoins(vCoins), out_set, out_value, coin_selection_params_bnb, bnb_used) || - testWallet.SelectCoinsMinConf(target, filter_standard, GroupCoins(vCoins), out_set, out_value, coin_selection_params_knapsack, bnb_used)); + BOOST_CHECK(testWallet.SelectCoinsMinConf(target, filter_standard, vCoins, out_set, out_value, coin_selection_params_bnb, bnb_used) || + testWallet.SelectCoinsMinConf(target, filter_standard, vCoins, out_set, out_value, coin_selection_params_knapsack, bnb_used)); BOOST_CHECK_GE(out_value, target); } } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 723552860a..db80745db0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2357,13 +2357,12 @@ const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int out return ptx->vout[n]; } -bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<OutputGroup> groups, +bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const { setCoinsRet.clear(); nValueRet = 0; - std::vector<OutputGroup> utxo_pool; if (coin_selection_params.use_bnb) { // Get long term estimate FeeCalculation feeCalc; @@ -2371,35 +2370,27 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibil temp.m_confirm_target = 1008; CFeeRate long_term_feerate = GetMinimumFeeRate(*this, temp, &feeCalc); - // Calculate cost of change - CAmount cost_of_change = GetDiscardRate(*this).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); + // Get the feerate for effective value. + // When subtracting the fee from the outputs, we want the effective feerate to be 0 + CFeeRate effective_feerate{0}; + if (!coin_selection_params.m_subtract_fee_outputs) { + effective_feerate = coin_selection_params.effective_fee; + } - // Filter by the min conf specs and add to utxo_pool and calculate effective value - for (OutputGroup& group : groups) { - if (!group.EligibleForSpending(eligibility_filter)) continue; + std::vector<OutputGroup> groups = GroupOutputs(coins, !coin_selection_params.m_avoid_partial_spends, effective_feerate, long_term_feerate, eligibility_filter, true /* positive_only */); - if (coin_selection_params.m_subtract_fee_outputs) { - // Set the effective feerate to 0 as we don't want to use the effective value since the fees will be deducted from the output - group.SetFees(CFeeRate(0) /* effective_feerate */, long_term_feerate); - } else { - group.SetFees(coin_selection_params.effective_fee, long_term_feerate); - } + // Calculate cost of change + CAmount cost_of_change = GetDiscardRate(*this).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); - OutputGroup pos_group = group.GetPositiveOnlyGroup(); - if (pos_group.effective_value > 0) utxo_pool.push_back(pos_group); - } // Calculate the fees for things that aren't inputs CAmount not_input_fees = coin_selection_params.effective_fee.GetFee(coin_selection_params.tx_noinputs_size); bnb_used = true; - return SelectCoinsBnB(utxo_pool, nTargetValue, cost_of_change, setCoinsRet, nValueRet, not_input_fees); + return SelectCoinsBnB(groups, nTargetValue, cost_of_change, setCoinsRet, nValueRet, not_input_fees); } else { - // Filter by the min conf specs and add to utxo_pool - for (const OutputGroup& group : groups) { - if (!group.EligibleForSpending(eligibility_filter)) continue; - utxo_pool.push_back(group); - } + std::vector<OutputGroup> groups = GroupOutputs(coins, !coin_selection_params.m_avoid_partial_spends, CFeeRate(0), CFeeRate(0), eligibility_filter, false /* positive_only */); + bnb_used = false; - return KnapsackSolver(nTargetValue, utxo_pool, setCoinsRet, nValueRet); + return KnapsackSolver(nTargetValue, groups, setCoinsRet, nValueRet); } } @@ -2482,16 +2473,14 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // explicitly shuffling the outputs before processing Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); } - std::vector<OutputGroup> groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends, max_ancestors); - bool res = value_to_select <= 0 || - SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max()), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); + SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || + SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 1, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, 2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // because SelectCoinsMinConf clears the setCoinsRet, we now add the possible inputs to the coinset util::insert(setCoinsRet, setPresetCoins); @@ -2782,6 +2771,7 @@ bool CWallet::CreateTransactionInternal( std::vector<COutput> vAvailableCoins; AvailableCoins(vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); CoinSelectionParams coin_selection_params; // Parameters for coin selection, init with dummy + coin_selection_params.m_avoid_partial_spends = coin_control.m_avoid_partial_spends; // Create change script that will be used if we need change // TODO: pass in scriptChange instead of reservedest so @@ -4192,51 +4182,90 @@ bool CWalletTx::IsImmatureCoinBase() const return GetBlocksToMaturity() > 0; } -std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const { - std::vector<OutputGroup> groups; - std::map<CTxDestination, OutputGroup> gmap; - std::set<CTxDestination> full_groups; +std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool separate_coins, const CFeeRate& effective_feerate, const CFeeRate& long_term_feerate, const CoinEligibilityFilter& filter, bool positive_only) const +{ + std::vector<OutputGroup> groups_out; - for (const auto& output : outputs) { - if (output.fSpendable) { - CTxDestination dst; - CInputCoin input_coin = output.GetInputCoin(); + if (separate_coins) { + // Single coin means no grouping. Each COutput gets its own OutputGroup. + for (const COutput& output : outputs) { + // Skip outputs we cannot spend + if (!output.fSpendable) continue; size_t ancestors, descendants; chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); - if (!single_coin && ExtractDestination(output.tx->tx->vout[output.i].scriptPubKey, dst)) { - auto it = gmap.find(dst); - if (it != gmap.end()) { - // Limit output groups to no more than OUTPUT_GROUP_MAX_ENTRIES - // number of entries, to protect against inadvertently creating - // a too-large transaction when using -avoidpartialspends to - // prevent breaking consensus or surprising users with a very - // high amount of fees. - if (it->second.m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { - groups.push_back(it->second); - it->second = OutputGroup{}; - full_groups.insert(dst); - } - it->second.Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); - } else { - gmap[dst].Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); - } - } else { - groups.emplace_back(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); - } + CInputCoin input_coin = output.GetInputCoin(); + + // Make an OutputGroup containing just this output + OutputGroup group{effective_feerate, long_term_feerate}; + group.Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants, positive_only); + + // Check the OutputGroup's eligibility. Only add the eligible ones. + if (positive_only && group.effective_value <= 0) continue; + if (group.m_outputs.size() > 0 && group.EligibleForSpending(filter)) groups_out.push_back(group); + } + return groups_out; + } + + // We want to combine COutputs that have the same scriptPubKey into single OutputGroups + // except when there are more than OUTPUT_GROUP_MAX_ENTRIES COutputs grouped in an OutputGroup. + // To do this, we maintain a map where the key is the scriptPubKey and the value is a vector of OutputGroups. + // For each COutput, we check if the scriptPubKey is in the map, and if it is, the COutput's CInputCoin is added + // to the last OutputGroup in the vector for the scriptPubKey. When the last OutputGroup has + // OUTPUT_GROUP_MAX_ENTRIES CInputCoins, a new OutputGroup is added to the end of the vector. + std::map<CScript, std::vector<OutputGroup>> spk_to_groups_map; + for (const auto& output : outputs) { + // Skip outputs we cannot spend + if (!output.fSpendable) continue; + + size_t ancestors, descendants; + chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); + CInputCoin input_coin = output.GetInputCoin(); + CScript spk = input_coin.txout.scriptPubKey; + + std::vector<OutputGroup>& groups = spk_to_groups_map[spk]; + + if (groups.size() == 0) { + // No OutputGroups for this scriptPubKey yet, add one + groups.emplace_back(effective_feerate, long_term_feerate); } + + // Get the last OutputGroup in the vector so that we can add the CInputCoin to it + // A pointer is used here so that group can be reassigned later if it is full. + OutputGroup* group = &groups.back(); + + // Check if this OutputGroup is full. We limit to OUTPUT_GROUP_MAX_ENTRIES when using -avoidpartialspends + // to avoid surprising users with very high fees. + if (group->m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { + // The last output group is full, add a new group to the vector and use that group for the insertion + groups.emplace_back(effective_feerate, long_term_feerate); + group = &groups.back(); + } + + // Add the input_coin to group + group->Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants, positive_only); } - if (!single_coin) { - for (auto& it : gmap) { - auto& group = it.second; - if (full_groups.count(it.first) > 0) { - // Make this unattractive as we want coin selection to avoid it if possible - group.m_ancestors = max_ancestors - 1; + + // Now we go through the entire map and pull out the OutputGroups + for (const auto& spk_and_groups_pair: spk_to_groups_map) { + const std::vector<OutputGroup>& groups_per_spk= spk_and_groups_pair.second; + + // Go through the vector backwards. This allows for the first item we deal with being the partial group. + for (auto group_it = groups_per_spk.rbegin(); group_it != groups_per_spk.rend(); group_it++) { + const OutputGroup& group = *group_it; + + // Don't include partial groups if there are full groups too and we don't want partial groups + if (group_it == groups_per_spk.rbegin() && groups_per_spk.size() > 1 && !filter.m_include_partial_groups) { + continue; } - groups.push_back(group); + + // Check the OutputGroup's eligibility. Only add the eligible ones. + if (positive_only && group.effective_value <= 0) continue; + if (group.m_outputs.size() > 0 && group.EligibleForSpending(filter)) groups_out.push_back(group); } } - return groups; + + return groups_out; } bool CWallet::IsCrypted() const diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index e3eae1dd95..4fc4466604 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -610,8 +610,16 @@ struct CoinSelectionParams size_t tx_noinputs_size = 0; //! Indicate that we are subtracting the fee from outputs bool m_subtract_fee_outputs = false; - - CoinSelectionParams(bool use_bnb, size_t change_output_size, size_t change_spend_size, CFeeRate effective_fee, size_t tx_noinputs_size) : use_bnb(use_bnb), change_output_size(change_output_size), change_spend_size(change_spend_size), effective_fee(effective_fee), tx_noinputs_size(tx_noinputs_size) {} + bool m_avoid_partial_spends = false; + + CoinSelectionParams(bool use_bnb, size_t change_output_size, size_t change_spend_size, CFeeRate effective_fee, size_t tx_noinputs_size, bool avoid_partial) : + use_bnb(use_bnb), + change_output_size(change_output_size), + change_spend_size(change_spend_size), + effective_fee(effective_fee), + tx_noinputs_size(tx_noinputs_size), + m_avoid_partial_spends(avoid_partial) + {} CoinSelectionParams() {} }; @@ -818,7 +826,7 @@ public: * completion the coin set and corresponding actual target value is * assembled */ - bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<OutputGroup> groups, + bool SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params, bool& bnb_used) const; bool IsSpent(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -827,7 +835,7 @@ public: bool IsSpentKey(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetSpentKeyState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const; + std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool separate_coins, const CFeeRate& effective_feerate, const CFeeRate& long_term_feerate, const CoinEligibilityFilter& filter, bool positive_only) const; bool IsLockedCoin(uint256 hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void LockCoin(const COutPoint& output) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index bc90491a2c..b2cb0bf479 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -103,10 +103,8 @@ static void WalletShowInfo(CWallet* wallet_instance) tfm::format(std::cout, "Address Book: %zu\n", wallet_instance->m_address_book.size()); } -bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, const std::string& name) +bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) { - const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), name); - if (args.IsArgSet("-format") && command != "createfromdump") { tfm::format(std::cerr, "The -format option can only be used with the \"createfromdump\" command.\n"); return false; @@ -119,6 +117,12 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, tfm::format(std::cerr, "The -descriptors option can only be used with the 'create' command.\n"); return false; } + if (command == "create" && !args.IsArgSet("-wallet")) { + tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); + return false; + } + const std::string name = args.GetArg("-wallet", ""); + const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), name); if (command == "create") { DatabaseOptions options; diff --git a/src/wallet/wallettool.h b/src/wallet/wallettool.h index f544a6f727..f4516bb5bc 100644 --- a/src/wallet/wallettool.h +++ b/src/wallet/wallettool.h @@ -10,7 +10,7 @@ namespace WalletTool { void WalletShowInfo(CWallet* wallet_instance); -bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command, const std::string& file); +bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command); } // namespace WalletTool diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index 1dd751b493..81859f924f 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -52,9 +52,9 @@ static RPCHelpMan getzmqnotifications() } const CRPCCommand commands[] = -{ // category name actor (function) argNames - // ----------------- ------------------------ ----------------------- ---------- - { "zmq", "getzmqnotifications", &getzmqnotifications, {} }, +{ // category actor (function) + // ----------------- ----------------------- + { "zmq", &getzmqnotifications, }, }; } // anonymous namespace diff --git a/test/functional/README.md b/test/functional/README.md index 2d04413eb2..d830ba0334 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -63,10 +63,13 @@ don't have test cases for. - Avoid stop-starting the nodes multiple times during the test if possible. A stop-start takes several seconds, so doing it several times blows up the runtime of the test. -- Set the `self.setup_clean_chain` variable in `set_test_params()` to control whether - or not to use the cached data directories. The cached data directories - contain a 200-block pre-mined blockchain and wallets for four nodes. Each node - has 25 mature blocks (25x50=1250 BTC) in its wallet. +- Set the `self.setup_clean_chain` variable in `set_test_params()` to `True` to + initialize an empty blockchain and start from the Genesis block, rather than + load a premined blockchain from cache with the default value of `False`. The + cached data directories contain a 200-block pre-mined blockchain with the + spendable mining rewards being split between four nodes. Each node has 25 + mature block subsidies (25x50=1250 BTC) in its wallet. Using them is much more + efficient than mining blocks in your test. - When calling RPCs with lots of arguments, consider using named keyword arguments instead of positional arguments to make the intent of the call clear to readers. diff --git a/test/functional/example_test.py b/test/functional/example_test.py index 97f24e1b6e..a0eb213a78 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -76,6 +76,9 @@ class ExampleTest(BitcoinTestFramework): """Override test parameters for your individual test. This method must be overridden and num_nodes must be explicitly set.""" + # By default every test loads a pre-mined chain of 200 blocks from cache. + # Set setup_clean_chain to True to skip this and start from the Genesis + # block. self.setup_clean_chain = True self.num_nodes = 3 # Use self.extra_args to change command-line arguments for the nodes diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index 2c6553fbe2..5fcecb4882 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -36,7 +36,6 @@ def expected_messages(filename): class AsmapTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def test_without_asmap_arg(self): diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index f9ece244fb..2b56bc78f5 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -49,7 +49,6 @@ from test_framework.util import ( class ChainstateWriteCrashTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 - self.setup_clean_chain = False self.rpc_timeout = 480 self.supports_cli = False diff --git a/test/functional/feature_includeconf.py b/test/functional/feature_includeconf.py index 6f1a0cd348..f22b7f266a 100755 --- a/test/functional/feature_includeconf.py +++ b/test/functional/feature_includeconf.py @@ -20,7 +20,6 @@ from test_framework.test_framework import BitcoinTestFramework class IncludeConfTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def setup_chain(self): diff --git a/test/functional/p2p_add_connections.py b/test/functional/p2p_add_connections.py index a63c3a3287..a04ba5db2d 100755 --- a/test/functional/p2p_add_connections.py +++ b/test/functional/p2p_add_connections.py @@ -17,7 +17,6 @@ def check_node_connections(*, node, num_in, num_out): class P2PAddConnections(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 def setup_network(self): diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 91fbd722cf..69821763bd 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -46,7 +46,6 @@ class AddrReceiver(P2PInterface): class AddrTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index c592ab52b1..6584efae79 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -15,7 +15,6 @@ from test_framework.util import assert_equal class P2PBlocksOnly(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 self.extra_args = [["-blocksonly"]] diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 642a217047..458e5235b6 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -81,7 +81,6 @@ class P2PBloomFilter(P2PInterface): class FilterTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 self.extra_args = [[ '-peerbloomfilters', diff --git a/test/functional/p2p_getaddr_caching.py b/test/functional/p2p_getaddr_caching.py index 2b75ad5175..d375af6fe1 100755 --- a/test/functional/p2p_getaddr_caching.py +++ b/test/functional/p2p_getaddr_caching.py @@ -41,7 +41,6 @@ class AddrReceiver(P2PInterface): class AddrTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index e4fc9fd178..f884cf90ff 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -13,7 +13,6 @@ from test_framework.test_framework import BitcoinTestFramework class InvalidLocatorTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - self.setup_clean_chain = False def run_test(self): node = self.nodes[0] # convenience reference to the node diff --git a/test/functional/p2p_message_capture.py b/test/functional/p2p_message_capture.py new file mode 100755 index 0000000000..113e26c425 --- /dev/null +++ b/test/functional/p2p_message_capture.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test per-peer message capture capability. + +Additionally, the output of contrib/message-capture/message-capture-parser.py should be verified manually. +""" + +import glob +from io import BytesIO +import os + +from test_framework.p2p import P2PDataStore, MESSAGEMAP +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +TIME_SIZE = 8 +LENGTH_SIZE = 4 +MSGTYPE_SIZE = 12 + +def mini_parser(dat_file): + """Parse a data file created by CaptureMessage. + + From the data file we'll only check the structure. + + We won't care about things like: + - Deserializing the payload of the message + - This is managed by the deserialize methods in test_framework.messages + - The order of the messages + - There's no reason why we can't, say, change the order of the messages in the handshake + - Message Type + - We can add new message types + + We're ignoring these because they're simply too brittle to test here. + """ + with open(dat_file, 'rb') as f_in: + # This should have at least one message in it + assert(os.fstat(f_in.fileno()).st_size >= TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + while True: + tmp_header_raw = f_in.read(TIME_SIZE + LENGTH_SIZE + MSGTYPE_SIZE) + if not tmp_header_raw: + break + tmp_header = BytesIO(tmp_header_raw) + int.from_bytes(tmp_header.read(TIME_SIZE), "little") # type: int + raw_msgtype = tmp_header.read(MSGTYPE_SIZE) + msgtype = raw_msgtype.split(b'\x00', 1)[0] # type: bytes + remainder = raw_msgtype.split(b'\x00', 1)[1] + assert(len(msgtype) > 0) + assert(msgtype in MESSAGEMAP) + assert(len(remainder) == 0 or not remainder.decode().isprintable()) + length = int.from_bytes(tmp_header.read(LENGTH_SIZE), "little") # type: int + data = f_in.read(length) + assert_equal(len(data), length) + + + +class MessageCaptureTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.extra_args = [["-capturemessages"]] + self.setup_clean_chain = True + + def run_test(self): + capturedir = os.path.join(self.nodes[0].datadir, "regtest/message_capture") + # Connect a node so that the handshake occurs + self.nodes[0].add_p2p_connection(P2PDataStore()) + self.nodes[0].disconnect_p2ps() + recv_file = glob.glob(os.path.join(capturedir, "*/msgs_recv.dat"))[0] + mini_parser(recv_file) + sent_file = glob.glob(os.path.join(capturedir, "*/msgs_sent.dat"))[0] + mini_parser(sent_file) + + +if __name__ == '__main__': + MessageCaptureTest().main() diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index 47832b04bf..a7e240dcfa 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -74,9 +74,9 @@ class TimeoutsTest(BitcoinTestFramework): no_version_node.send_message(msg_ping()) expected_timeout_logs = [ - "version handshake timeout from 0", - "socket no message in first 3 seconds, 1 0 from 1", - "socket no message in first 3 seconds, 0 0 from 2", + "version handshake timeout peer=0", + "socket no message in first 3 seconds, 1 0 peer=1", + "socket no message in first 3 seconds, 0 0 peer=2", ] with self.nodes[0].assert_debug_log(expected_msgs=expected_timeout_logs): diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 8a751c6b54..4bf96cb0e6 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -56,7 +56,6 @@ MAX_GETDATA_INBOUND_WAIT = GETDATA_TX_INTERVAL + INBOUND_PEER_TX_DELAY + TXID_RE class TxDownloadTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 def test_tx_requests(self): diff --git a/test/functional/rpc_estimatefee.py b/test/functional/rpc_estimatefee.py index 81862ac69e..51b7efb4c3 100755 --- a/test/functional/rpc_estimatefee.py +++ b/test/functional/rpc_estimatefee.py @@ -14,7 +14,6 @@ from test_framework.util import assert_raises_rpc_error class EstimateFeeTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def run_test(self): diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index c83157a843..1eefd109f8 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -7,7 +7,39 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error +from collections import defaultdict import os +import re + + +def parse_string(s): + assert s[0] == '"' + assert s[-1] == '"' + return s[1:-1] + + +def process_mapping(fname): + """Find and parse conversion table in implementation file `fname`.""" + cmds = [] + in_rpcs = False + with open(fname, "r", encoding="utf8") as f: + for line in f: + line = line.rstrip() + if not in_rpcs: + if line == 'static const CRPCConvertParam vRPCConvertParams[] =': + in_rpcs = True + else: + if line.startswith('};'): + in_rpcs = False + elif '{' in line and '"' in line: + m = re.search(r'{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line) + assert m, 'No match to table expression: %s' % line + name = parse_string(m.group(1)) + idx = int(m.group(2)) + argname = parse_string(m.group(3)) + cmds.append((name, idx, argname)) + assert not in_rpcs and cmds + return cmds class HelpRpcTest(BitcoinTestFramework): @@ -16,11 +48,43 @@ class HelpRpcTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): + self.test_client_conversion_table() self.test_categories() self.dump_help() if self.is_wallet_compiled(): self.wallet_help() + def test_client_conversion_table(self): + file_conversion_table = os.path.join(self.config["environment"]["SRCDIR"], 'src', 'rpc', 'client.cpp') + mapping_client = process_mapping(file_conversion_table) + # Ignore echojson in client table + mapping_client = [m for m in mapping_client if m[0] != 'echojson'] + + mapping_server = self.nodes[0].help("dump_all_command_conversions") + # Filter all RPCs whether they need conversion + mapping_server_conversion = [tuple(m[:3]) for m in mapping_server if not m[3]] + + # Only check if all RPC methods have been compiled (i.e. wallet is enabled) + if self.is_wallet_compiled() and sorted(mapping_client) != sorted(mapping_server_conversion): + raise AssertionError("RPC client conversion table ({}) and RPC server named arguments mismatch!\n{}".format( + file_conversion_table, + set(mapping_client).symmetric_difference(mapping_server_conversion), + )) + + # Check for conversion difference by argument name. + # It is preferable for API consistency that arguments with the same name + # have the same conversion, so bin by argument name. + all_methods_by_argname = defaultdict(list) + converts_by_argname = defaultdict(list) + for m in mapping_server: + all_methods_by_argname[m[2]].append(m[0]) + converts_by_argname[m[2]].append(m[3]) + + for argname, convert in converts_by_argname.items(): + if all(convert) != any(convert): + # Only allow dummy to fail consistency check + assert argname == 'dummy', ('WARNING: conversion mismatch for argument named %s (%s)' % (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) + def test_categories(self): node = self.nodes[0] diff --git a/test/functional/rpc_invalid_address_message.py b/test/functional/rpc_invalid_address_message.py new file mode 100755 index 0000000000..469d6bdb05 --- /dev/null +++ b/test/functional/rpc_invalid_address_message.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test error messages for 'getaddressinfo' and 'validateaddress' RPC commands.""" + +from test_framework.test_framework import BitcoinTestFramework + +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +BECH32_VALID = 'bcrt1qtmp74ayg7p24uslctssvjm06q5phz4yrxucgnv' +BECH32_INVALID_SIZE = 'bcrt1sqqpl9r5c' +BECH32_INVALID_PREFIX = 'bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4' + +BASE58_VALID = 'mipcBbFg9gMiCh81Kj8tqqdgoZub1ZJRfn' +BASE58_INVALID_PREFIX = '17VZNX1SN5NtKa8UQFxwQbFeFc3iqRYhem' + +INVALID_ADDRESS = 'asfah14i8fajz0123f' + +class InvalidAddressErrorMessageTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def test_validateaddress(self): + node = self.nodes[0] + + # Bech32 + info = node.validateaddress(BECH32_INVALID_SIZE) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid Bech32 address data size') + + info = node.validateaddress(BECH32_INVALID_PREFIX) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid prefix for Bech32 address') + + info = node.validateaddress(BECH32_VALID) + assert info['isvalid'] + assert 'error' not in info + + # Base58 + info = node.validateaddress(BASE58_INVALID_PREFIX) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid prefix for Base58-encoded address') + + info = node.validateaddress(BASE58_VALID) + assert info['isvalid'] + assert 'error' not in info + + # Invalid address format + info = node.validateaddress(INVALID_ADDRESS) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid address format') + + def test_getaddressinfo(self): + node = self.nodes[0] + + assert_raises_rpc_error(-5, "Invalid Bech32 address data size", node.getaddressinfo, BECH32_INVALID_SIZE) + + assert_raises_rpc_error(-5, "Invalid prefix for Bech32 address", node.getaddressinfo, BECH32_INVALID_PREFIX) + + assert_raises_rpc_error(-5, "Invalid prefix for Base58-encoded address", node.getaddressinfo, BASE58_INVALID_PREFIX) + + assert_raises_rpc_error(-5, "Invalid address format", node.getaddressinfo, INVALID_ADDRESS) + + def run_test(self): + self.test_validateaddress() + self.test_getaddressinfo() + + +if __name__ == '__main__': + InvalidAddressErrorMessageTest().main() diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index b364077a9a..ed6abaed78 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -24,7 +24,6 @@ MAX_BIP125_RBF_SEQUENCE = 0xfffffffd class PSBTTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 3 self.extra_args = [ ["-walletrbf=1"], diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md index f6ea9ef682..b8e899d675 100644 --- a/test/functional/test-shell.md +++ b/test/functional/test-shell.md @@ -178,7 +178,7 @@ can be called after the TestShell is shut down. | `num_nodes` | `1` | Sets the number of initialized bitcoind processes. | | `perf` | False | Profiles running nodes with `perf` for the duration of the test if set to `True`. | | `rpc_timeout` | `60` | Sets the RPC server timeout for the underlying bitcoind processes. | -| `setup_clean_chain` | `False` | Initializes an empty blockchain by default. A 199-block-long chain is initialized if set to `True`. | +| `setup_clean_chain` | `False` | A 200-block-long chain is initialized from cache by default. Instead, `setup_clean_chain` initializes an empty blockchain if set to `True`. | | `randomseed` | Random Integer | `TestShell.options.randomseed` is a member of `TestShell` which can be accessed during a test to seed a random generator. User can override default with a constant value for reproducible test runs. | | `supports_cli` | `False` | Whether the bitcoin-cli utility is compiled and available for the test. | | `tmpdir` | `"/var/folders/.../"` | Sets directory for test logs. Will be deleted upon a successful test run unless `nocleanup` is set to `True` | diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 6ad4e13db2..27a09ef86c 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1273,7 +1273,7 @@ class msg_block: # for cases where a user needs tighter control over what is sent over the wire # note that the user must supply the name of the msgtype, and the data class msg_generic: - __slots__ = ("msgtype", "data") + __slots__ = ("data") def __init__(self, msgtype, data=None): self.msgtype = msgtype diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 1f5dc0a6bf..70a9798449 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -18,6 +18,7 @@ import sys import tempfile import time +from typing import List from .authproxy import JSONRPCException from . import coverage from .p2p import NetworkThread @@ -89,14 +90,11 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): This class also contains various public and private helper methods.""" - chain = None # type: str - setup_clean_chain = None # type: bool - def __init__(self): """Sets test framework defaults. Do not override this method. Instead, override the set_test_params() method""" - self.chain = 'regtest' - self.setup_clean_chain = False - self.nodes = [] + self.chain: str = 'regtest' + self.setup_clean_chain: bool = False + self.nodes: List[TestNode] = [] self.network_thread = None self.rpc_timeout = 60 # Wait for up to 60 seconds for the RPC server to respond self.supports_cli = True diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index a9726cee8a..1e7d9e4b0d 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -134,6 +134,7 @@ BASE_SCRIPTS = [ 'wallet_keypool_topup.py --descriptors', 'feature_fee_estimation.py', 'interface_zmq.py', + 'rpc_invalid_address_message.py', 'interface_bitcoin_cli.py', 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', @@ -237,6 +238,7 @@ BASE_SCRIPTS = [ 'rpc_named_arguments.py', 'wallet_listsinceblock.py --legacy-wallet', 'wallet_listsinceblock.py --descriptors', + 'wallet_listdescriptors.py --descriptors', 'p2p_leak.py', 'wallet_encryption.py --legacy-wallet', 'wallet_encryption.py --descriptors', @@ -265,6 +267,7 @@ BASE_SCRIPTS = [ 'p2p_add_connections.py', 'p2p_unrequested_blocks.py', 'p2p_blockfilters.py', + 'p2p_message_capture.py', 'feature_includeconf.py', 'feature_asmap.py', 'mempool_unbroadcast.py', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 232848c7cf..28103793df 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -183,11 +183,13 @@ class ToolWalletTest(BitcoinTestFramework): def test_invalid_tool_commands_and_args(self): self.log.info('Testing that various invalid commands raise with specific error messages') - self.assert_raises_tool_error('Invalid command: foo', 'foo') + self.assert_raises_tool_error("Error parsing command line arguments: Invalid command 'foo'", 'foo') # `bitcoin-wallet help` raises an error. Use `bitcoin-wallet -help`. - self.assert_raises_tool_error('Invalid command: help', 'help') - self.assert_raises_tool_error('Error: two methods provided (info and create). Only one method should be provided.', 'info', 'create') + self.assert_raises_tool_error("Error parsing command line arguments: Invalid command 'help'", 'help') + self.assert_raises_tool_error('Error: Additional arguments provided (create). Methods do not take arguments. Please refer to `-help`.', 'info', 'create') self.assert_raises_tool_error('Error parsing command line arguments: Invalid parameter -foo', '-foo') + self.assert_raises_tool_error('No method provided. Run `bitcoin-wallet -help` for valid methods.') + self.assert_raises_tool_error('Wallet name must be provided when creating a new wallet.', 'create') locked_dir = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets") error = 'Error initializing wallet database environment "{}"!'.format(locked_dir) if self.options.descriptors: diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 229c134a4b..bc4fa90e83 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -65,7 +65,6 @@ def assert_balances(node, mine): class AvoidReuseTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 2 # This test isn't testing txn relay/timing, so set whitelist on the # peers for instant txn relay. This speeds up the test run time 2-3x. diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 6bcb03e8ed..4a589f0393 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -586,7 +586,7 @@ class WalletTest(BitcoinTestFramework): assert_equal(total_txs, len(self.nodes[0].listtransactions("*", 99999))) # Test getaddressinfo on external address. Note that these addresses are taken from disablewallet.py - assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].getaddressinfo, "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy") + assert_raises_rpc_error(-5, "Invalid prefix for Base58-encoded address", self.nodes[0].getaddressinfo, "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy") address_info = self.nodes[0].getaddressinfo("mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ") assert_equal(address_info['address'], "mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ") assert_equal(address_info["scriptPubKey"], "76a9144e3854046c7bd1594ac904e4793b6a45b36dea0988ac") diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index cf3317121f..16a0a50b07 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -17,7 +17,6 @@ from test_framework.wallet_util import bytes_to_wif, generate_wif_key class CreateWalletTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def skip_test_if_missing_module(self): diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py new file mode 100755 index 0000000000..9f8c341bc7 --- /dev/null +++ b/test/functional/wallet_listdescriptors.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the listdescriptors RPC.""" + +from test_framework.descriptors import ( + descsum_create +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + + +class ListDescriptorsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_sqlite() + + # do not create any wallet by default + def init_wallet(self, i): + return + + def run_test(self): + node = self.nodes[0] + assert_raises_rpc_error(-18, 'No wallet is loaded.', node.listdescriptors) + + self.log.info('Test that the command is not available for legacy wallets.') + node.createwallet(wallet_name='w1', descriptors=False) + assert_raises_rpc_error(-4, 'listdescriptors is not available for non-descriptor wallets', node.listdescriptors) + + self.log.info('Test the command for empty descriptors wallet.') + node.createwallet(wallet_name='w2', blank=True, descriptors=True) + assert_equal(0, len(node.get_wallet_rpc('w2').listdescriptors())) + + self.log.info('Test the command for a default descriptors wallet.') + node.createwallet(wallet_name='w3', descriptors=True) + result = node.get_wallet_rpc('w3').listdescriptors() + assert_equal(6, len(result)) + assert_equal(6, len([d for d in result if d['active']])) + assert_equal(3, len([d for d in result if d['internal']])) + for item in result: + assert item['desc'] != '' + assert item['next'] == 0 + assert item['range'] == [0, 0] + assert item['timestamp'] is not None + + self.log.info('Test non-active non-range combo descriptor') + node.createwallet(wallet_name='w4', blank=True, descriptors=True) + wallet = node.get_wallet_rpc('w4') + wallet.importdescriptors([{ + 'desc': descsum_create('combo(' + node.get_deterministic_priv_key().key + ')'), + 'timestamp': 1296688602, + }]) + expected = [{'active': False, + 'desc': 'combo(0227d85ba011276cf25b51df6a188b75e604b38770a462b2d0e9fb2fc839ef5d3f)#np574htj', + 'timestamp': 1296688602}] + assert_equal(expected, wallet.listdescriptors()) + + +if __name__ == '__main__': + ListDescriptorsTest().main() diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index bb89e76a9a..bf24b9c7b3 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -304,12 +304,12 @@ class MultiWalletTest(BitcoinTestFramework): if self.options.descriptors: assert_raises_rpc_error(-4, "Wallet file verification failed. SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?", self.nodes[0].loadwallet, wallet_names[0]) else: - assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0]) + assert_raises_rpc_error(-35, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0]) # This tests the default wallet that BDB makes, so SQLite wallet doesn't need to test this # Fail to load duplicate wallets by different ways (directory and filepath) path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "wallet.dat") - assert_raises_rpc_error(-4, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat') + assert_raises_rpc_error(-35, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, 'wallet.dat') # Only BDB doesn't open duplicate wallet files. SQLite does not have this limitation. While this may be desired in the future, it is not necessary # Fail to load if one wallet is a copy of another diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index b0c41b2738..24799fe5f2 100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py @@ -14,7 +14,6 @@ from test_framework.util import ( class CreateWalletWatchonlyTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False self.num_nodes = 1 def skip_test_if_missing_module(self): diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 3c743603bb..aa0aa11d15 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -14,6 +14,16 @@ import subprocess import sys +def get_fuzz_env(*, target, source_dir): + return { + 'FUZZ': target, + 'UBSAN_OPTIONS': + f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1', + 'ASAN_OPTIONS': # symbolizer disabled due to https://github.com/google/sanitizers/issues/1364#issuecomment-761072085 + 'symbolize=0:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1', + } + + def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, @@ -129,9 +139,7 @@ def main(): os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), '-help=1', ], - env={ - 'FUZZ': test_list_selection[0] - }, + env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']), timeout=20, check=True, stderr=subprocess.PIPE, @@ -148,6 +156,7 @@ def main(): if args.generate: return generate_corpus_seeds( fuzz_pool=fuzz_pool, + src_dir=config['environment']['SRCDIR'], build_dir=config["environment"]["BUILDDIR"], seed_dir=args.seed_dir, targets=test_list_selection, @@ -158,6 +167,7 @@ def main(): fuzz_pool=fuzz_pool, corpus=args.seed_dir, test_list=test_list_selection, + src_dir=config['environment']['SRCDIR'], build_dir=config["environment"]["BUILDDIR"], merge_dir=args.m_dir, ) @@ -167,12 +177,13 @@ def main(): fuzz_pool=fuzz_pool, corpus=args.seed_dir, test_list=test_list_selection, + src_dir=config['environment']['SRCDIR'], build_dir=config["environment"]["BUILDDIR"], use_valgrind=args.valgrind, ) -def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): +def generate_corpus_seeds(*, fuzz_pool, src_dir, build_dir, seed_dir, targets): """Generates new corpus seeds. Run {targets} without input, and outputs the generated corpus seeds to @@ -186,9 +197,7 @@ def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): ' '.join(command), subprocess.run( command, - env={ - 'FUZZ': t - }, + env=get_fuzz_env(target=t, source_dir=src_dir), check=True, stderr=subprocess.PIPE, universal_newlines=True, @@ -209,7 +218,7 @@ def generate_corpus_seeds(*, fuzz_pool, build_dir, seed_dir, targets): future.result() -def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): +def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dir): logging.info("Merge the inputs from the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) jobs = [] for t in test_list: @@ -227,9 +236,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): output = 'Run {} with args {}\n'.format(t, " ".join(args)) output += subprocess.run( args, - env={ - 'FUZZ': t - }, + env=get_fuzz_env(target=t, source_dir=src_dir), check=True, stderr=subprocess.PIPE, universal_newlines=True, @@ -242,7 +249,7 @@ def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): future.result() -def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): +def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, use_valgrind): jobs = [] for t in test_list: corpus_path = os.path.join(corpus, t) @@ -257,7 +264,12 @@ def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): def job(t, args): output = 'Run {} with args {}'.format(t, args) - result = subprocess.run(args, env={'FUZZ': t}, stderr=subprocess.PIPE, universal_newlines=True) + result = subprocess.run( + args, + env=get_fuzz_env(target=t, source_dir=src_dir), + stderr=subprocess.PIPE, + universal_newlines=True, + ) output += result.stderr return output, result diff --git a/test/lint/check-rpc-mappings.py b/test/lint/check-rpc-mappings.py deleted file mode 100755 index 0a4cc875d0..0000000000 --- a/test/lint/check-rpc-mappings.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Check RPC argument consistency.""" - -from collections import defaultdict -import os -import re -import sys - -# Source files (relative to root) to scan for dispatch tables -SOURCES = [ - "src/rpc/server.cpp", - "src/rpc/blockchain.cpp", - "src/rpc/mining.cpp", - "src/rpc/misc.cpp", - "src/rpc/net.cpp", - "src/rpc/rawtransaction.cpp", - "src/wallet/rpcwallet.cpp", -] -# Source file (relative to root) containing conversion mapping -SOURCE_CLIENT = 'src/rpc/client.cpp' -# Argument names that should be ignored in consistency checks -IGNORE_DUMMY_ARGS = {'dummy', 'arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'arg5', 'arg6', 'arg7', 'arg8', 'arg9'} - -class RPCCommand: - def __init__(self, name, args): - self.name = name - self.args = args - -class RPCArgument: - def __init__(self, names, idx): - self.names = names - self.idx = idx - self.convert = False - -def parse_string(s): - assert s[0] == '"' - assert s[-1] == '"' - return s[1:-1] - -def process_commands(fname): - """Find and parse dispatch table in implementation file `fname`.""" - cmds = [] - in_rpcs = False - with open(fname, "r", encoding="utf8") as f: - for line in f: - line = line.rstrip() - if not in_rpcs: - if re.match(r"static const CRPCCommand .*\[\] =", line): - in_rpcs = True - else: - if line.startswith('};'): - in_rpcs = False - elif '{' in line and '"' in line: - m = re.search(r'{ *("[^"]*"), *("[^"]*"), *&([^,]*), *{([^}]*)} *},', line) - assert m, 'No match to table expression: %s' % line - name = parse_string(m.group(2)) - args_str = m.group(4).strip() - if args_str: - args = [RPCArgument(parse_string(x.strip()).split('|'), idx) for idx, x in enumerate(args_str.split(','))] - else: - args = [] - cmds.append(RPCCommand(name, args)) - assert not in_rpcs and cmds, "Something went wrong with parsing the C++ file: update the regexps" - return cmds - -def process_mapping(fname): - """Find and parse conversion table in implementation file `fname`.""" - cmds = [] - in_rpcs = False - with open(fname, "r", encoding="utf8") as f: - for line in f: - line = line.rstrip() - if not in_rpcs: - if line == 'static const CRPCConvertParam vRPCConvertParams[] =': - in_rpcs = True - else: - if line.startswith('};'): - in_rpcs = False - elif '{' in line and '"' in line: - m = re.search(r'{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line) - assert m, 'No match to table expression: %s' % line - name = parse_string(m.group(1)) - idx = int(m.group(2)) - argname = parse_string(m.group(3)) - cmds.append((name, idx, argname)) - assert not in_rpcs and cmds - return cmds - -def main(): - if len(sys.argv) != 2: - print('Usage: {} ROOT-DIR'.format(sys.argv[0]), file=sys.stderr) - sys.exit(1) - - root = sys.argv[1] - - # Get all commands from dispatch tables - cmds = [] - for fname in SOURCES: - cmds += process_commands(os.path.join(root, fname)) - - cmds_by_name = {} - for cmd in cmds: - cmds_by_name[cmd.name] = cmd - - # Get current convert mapping for client - client = SOURCE_CLIENT - mapping = set(process_mapping(os.path.join(root, client))) - - print('* Checking consistency between dispatch tables and vRPCConvertParams') - - # Check mapping consistency - errors = 0 - for (cmdname, argidx, argname) in mapping: - try: - rargnames = cmds_by_name[cmdname].args[argidx].names - except IndexError: - print('ERROR: %s argument %i (named %s in vRPCConvertParams) is not defined in dispatch table' % (cmdname, argidx, argname)) - errors += 1 - continue - if argname not in rargnames: - print('ERROR: %s argument %i is named %s in vRPCConvertParams but %s in dispatch table' % (cmdname, argidx, argname, rargnames), file=sys.stderr) - errors += 1 - - # Check for conflicts in vRPCConvertParams conversion - # All aliases for an argument must either be present in the - # conversion table, or not. Anything in between means an oversight - # and some aliases won't work. - for cmd in cmds: - for arg in cmd.args: - convert = [((cmd.name, arg.idx, argname) in mapping) for argname in arg.names] - if any(convert) != all(convert): - print('ERROR: %s argument %s has conflicts in vRPCConvertParams conversion specifier %s' % (cmd.name, arg.names, convert)) - errors += 1 - arg.convert = all(convert) - - # Check for conversion difference by argument name. - # It is preferable for API consistency that arguments with the same name - # have the same conversion, so bin by argument name. - all_methods_by_argname = defaultdict(list) - converts_by_argname = defaultdict(list) - for cmd in cmds: - for arg in cmd.args: - for argname in arg.names: - all_methods_by_argname[argname].append(cmd.name) - converts_by_argname[argname].append(arg.convert) - - for argname, convert in converts_by_argname.items(): - if all(convert) != any(convert): - if argname in IGNORE_DUMMY_ARGS: - # these are testing or dummy, don't warn for them - continue - print('WARNING: conversion mismatch for argument named %s (%s)' % - (argname, list(zip(all_methods_by_argname[argname], converts_by_argname[argname])))) - - sys.exit(errors > 0) - - -if __name__ == '__main__': - main() diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index dc032665e4..fc0b86a297 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -60,17 +60,13 @@ EXPECTED_BOOST_INCLUDES=( boost/multi_index/ordered_index.hpp boost/multi_index/sequenced_index.hpp boost/multi_index_container.hpp - boost/preprocessor/cat.hpp - boost/preprocessor/stringize.hpp boost/process.hpp boost/signals2/connection.hpp boost/signals2/optional_last_value.hpp boost/signals2/signal.hpp boost/test/unit_test.hpp - boost/thread/condition_variable.hpp - boost/thread/mutex.hpp + boost/thread/lock_types.hpp boost/thread/shared_mutex.hpp - boost/thread/thread.hpp ) for BOOST_INCLUDE in $(git grep '^#include <boost/' -- "*.cpp" "*.h" | cut -f2 -d: | cut -f2 -d'<' | cut -f1 -d'>' | sort -u); do diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 9a52cd4b57..97f0f45e7f 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -1,3 +1,7 @@ +# -fsanitize=undefined suppressions +# ================================= +# No suppressions at the moment. Hooray! + # -fsanitize=integer suppressions # =============================== # Unsigned integer overflow occurs when the result of an unsigned integer @@ -6,7 +10,8 @@ # contains files in which we expect unsigned integer overflows to occur. The # list is used to suppress -fsanitize=integer warnings when running our CI UBSan # job. -unsigned-integer-overflow:*/include/c++/*/bits/basic_string.tcc +unsigned-integer-overflow:*/include/c++/ +unsigned-integer-overflow:addrman.cpp unsigned-integer-overflow:arith_uint256.h unsigned-integer-overflow:basic_string.h unsigned-integer-overflow:bench/bench.h @@ -15,34 +20,41 @@ unsigned-integer-overflow:bloom.cpp unsigned-integer-overflow:chain.cpp unsigned-integer-overflow:chain.h unsigned-integer-overflow:coded_stream.h +unsigned-integer-overflow:coins.cpp +unsigned-integer-overflow:compressor.cpp unsigned-integer-overflow:core_write.cpp -unsigned-integer-overflow:crypto/* +unsigned-integer-overflow:crypto/ +# unsigned-integer-overflow in FuzzedDataProvider's ConsumeIntegralInRange +unsigned-integer-overflow:FuzzedDataProvider.h unsigned-integer-overflow:hash.cpp -unsigned-integer-overflow:leveldb/db/log_reader.cc -unsigned-integer-overflow:leveldb/util/bloom.cc -unsigned-integer-overflow:leveldb/util/crc32c.h -unsigned-integer-overflow:leveldb/util/hash.cc +unsigned-integer-overflow:leveldb/ unsigned-integer-overflow:policy/fees.cpp unsigned-integer-overflow:prevector.h +unsigned-integer-overflow:pubkey.h unsigned-integer-overflow:script/interpreter.cpp unsigned-integer-overflow:stl_bvector.h unsigned-integer-overflow:txmempool.cpp unsigned-integer-overflow:util/strencodings.cpp unsigned-integer-overflow:validation.cpp - -implicit-integer-sign-change:*/include/c++/*/bits/*.h +implicit-integer-sign-change:*/include/boost/ +implicit-integer-sign-change:*/include/c++/ implicit-integer-sign-change:*/new_allocator.h -implicit-integer-sign-change:/usr/include/boost/date_time/format_date_parser.hpp +implicit-integer-sign-change:addrman.h implicit-integer-sign-change:arith_uint256.cpp implicit-integer-sign-change:bech32.cpp implicit-integer-sign-change:bloom.cpp -implicit-integer-sign-change:chain.* +implicit-integer-sign-change:chain.cpp +implicit-integer-sign-change:chain.h implicit-integer-sign-change:coins.h implicit-integer-sign-change:compat/stdin.cpp implicit-integer-sign-change:compressor.h -implicit-integer-sign-change:crypto/* +implicit-integer-sign-change:crc32c/ +implicit-integer-sign-change:crypto/ +# implicit-integer-sign-change in FuzzedDataProvider's ConsumeIntegralInRange +implicit-integer-sign-change:FuzzedDataProvider.h implicit-integer-sign-change:key.cpp implicit-integer-sign-change:noui.cpp +implicit-integer-sign-change:policy/fees.cpp implicit-integer-sign-change:prevector.h implicit-integer-sign-change:protocol.cpp implicit-integer-sign-change:script/bitcoinconsensus.cpp @@ -53,24 +65,38 @@ implicit-integer-sign-change:test/coins_tests.cpp implicit-integer-sign-change:test/pow_tests.cpp implicit-integer-sign-change:test/prevector_tests.cpp implicit-integer-sign-change:test/sighash_tests.cpp +implicit-integer-sign-change:test/skiplist_tests.cpp implicit-integer-sign-change:test/streams_tests.cpp implicit-integer-sign-change:test/transaction_tests.cpp implicit-integer-sign-change:txmempool.cpp -implicit-integer-sign-change:util/strencodings.* +implicit-integer-sign-change:util/strencodings.cpp +implicit-integer-sign-change:util/strencodings.h implicit-integer-sign-change:validation.cpp implicit-integer-sign-change:zmq/zmqpublishnotifier.cpp implicit-signed-integer-truncation,implicit-integer-sign-change:chain.h implicit-signed-integer-truncation,implicit-integer-sign-change:test/skiplist_tests.cpp +implicit-signed-integer-truncation:addrman.cpp +implicit-signed-integer-truncation:addrman.h implicit-signed-integer-truncation:chain.h -implicit-signed-integer-truncation:crypto/* +implicit-signed-integer-truncation:crypto/ implicit-signed-integer-truncation:cuckoocache.h -implicit-signed-integer-truncation:leveldb/* +implicit-signed-integer-truncation:leveldb/ +implicit-signed-integer-truncation:net.cpp +implicit-signed-integer-truncation:net_processing.cpp implicit-signed-integer-truncation:streams.h implicit-signed-integer-truncation:test/arith_uint256_tests.cpp implicit-signed-integer-truncation:test/skiplist_tests.cpp implicit-signed-integer-truncation:torcontrol.cpp -implicit-unsigned-integer-truncation:crypto/* -implicit-unsigned-integer-truncation:leveldb/* +implicit-unsigned-integer-truncation:*/include/c++/ +implicit-unsigned-integer-truncation:crypto/ +implicit-unsigned-integer-truncation:leveldb/ # std::variant warning fixed in https://github.com/gcc-mirror/gcc/commit/074436cf8cdd2a9ce75cadd36deb8301f00e55b9 implicit-unsigned-integer-truncation:std::__detail::__variant::_Variant_storage -implicit-integer-sign-change:crc32c/* +shift-base:*/include/c++/ +shift-base:arith_uint256.cpp +shift-base:crypto/ +shift-base:hash.cpp +shift-base:leveldb/ +shift-base:net_processing.cpp +shift-base:streams.h +shift-base:util/bip32.cpp |