diff options
184 files changed, 3519 insertions, 2158 deletions
diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index fa9bf4c646..f174b4d074 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -22,7 +22,7 @@ test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c test/lint/check-doc.py -test/lint/lint-all.sh +test/lint/lint-all.py if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then # Sanity check only the last few commits to get notified of missing sigs, diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 763a5e8b6f..63560a5f5c 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -11,4 +11,4 @@ export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tes export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" # Use clang-8 to test C++17 compatibility, see doc/dependencies.md export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate" +export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 87dd315e2e..e4d3468473 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export DOCKER_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_tidy -export PACKAGES="clang llvm clang-tidy bear libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export PACKAGES="clang libclang-dev llvm-dev clang-tidy bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index c1324a0b14..227bdf3319 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -111,6 +111,13 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" fi +if [[ "${RUN_TIDY}" == "true" ]]; then + CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/iwyu/build/" + CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_14 ${BASE_SCRATCH_DIR}/iwyu/include-what-you-use" + CI_EXEC "cd ${BASE_SCRATCH_DIR}/iwyu/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-14 ../include-what-you-use" + CI_EXEC "cd ${BASE_SCRATCH_DIR}/iwyu/build && make install $MAKEJOBS" +fi + if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR" diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 4742cb02ea..6a6bde05a1 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -49,7 +49,7 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then fi if [[ "${RUN_TIDY}" == "true" ]]; then - MAYBE_BEAR="bear" + MAYBE_BEAR="bear --config src/.bear-tidy-config" MAYBE_TOKEN="--" fi diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 30788f1543..afab4c5e78 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -37,6 +37,8 @@ fi if [ "${RUN_TIDY}" = "true" ]; then export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/src/" CI_EXEC run-clang-tidy "${MAKEJOBS}" + export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST/" + CI_EXEC "python3 ${BASE_SCRATCH_DIR}/iwyu/include-what-you-use/iwyu_tool.py src/compat src/init -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" fi if [ "$RUN_SECURITY_TESTS" = "true" ]; then diff --git a/configure.ac b/configure.ac index 4c80899e5f..7f1ed63ddf 100644 --- a/configure.ac +++ b/configure.ac @@ -663,6 +663,12 @@ AC_ARG_WITH([libs], [build_bitcoin_libs=$withval], [build_bitcoin_libs=yes]) +AC_ARG_WITH([experimental-kernel-lib], + [AS_HELP_STRING([--with-experimental-kernel-lib], + [build experimental bitcoinkernel library (default is to build if we're building libraries and the experimental build-chainstate executable)])], + [build_experimental_kernel_lib=$withval], + [build_experimental_kernel_lib=auto]) + AC_ARG_WITH([daemon], [AS_HELP_STRING([--with-daemon], [build bitcoind daemon (default=yes)])], @@ -1656,15 +1662,23 @@ AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test $build_bitcoin_util = "yes"]) AC_MSG_RESULT($build_bitcoin_util) AC_MSG_CHECKING([whether to build experimental bitcoin-chainstate]) -AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"]) +if test "$build_experimental_kernel_lib" = "no"; then +AC_MSG_ERROR([experimental bitcoin-chainstate cannot be built without the experimental bitcoinkernel library. Use --with-experimental-kernel-lib]); +else + AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"]) +fi AC_MSG_RESULT($build_bitcoin_chainstate) AC_MSG_CHECKING([whether to build libraries]) AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"]) + if test "$build_bitcoin_libs" = "yes"; then AC_DEFINE([HAVE_CONSENSUS_LIB], [1], [Define this symbol if the consensus lib has been built]) AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) fi + +AM_CONDITIONAL([BUILD_BITCOIN_KERNEL_LIB], [test "$build_experimental_kernel_lib" != "no" && ( test "$build_experimental_kernel_lib" = "yes" || test "$build_bitcoin_chainstate" = "yes" )]) + AC_MSG_RESULT($build_bitcoin_libs) AC_LANG_POP @@ -1933,8 +1947,10 @@ AC_CONFIG_LINKS([contrib/devtools/security-check.py:contrib/devtools/security-ch AC_CONFIG_LINKS([contrib/devtools/symbol-check.py:contrib/devtools/symbol-check.py]) AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-security-check.py]) AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-symbol-check.py]) +AC_CONFIG_LINKS([contrib/devtools/iwyu/bitcoin.core.imp:contrib/devtools/iwyu/bitcoin.core.imp]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) AC_CONFIG_LINKS([contrib/macdeploy/background.tiff:contrib/macdeploy/background.tiff]) +AC_CONFIG_LINKS([src/.bear-tidy-config:src/.bear-tidy-config]) AC_CONFIG_LINKS([src/.clang-tidy:src/.clang-tidy]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) diff --git a/contrib/devtools/iwyu/bitcoin.core.imp b/contrib/devtools/iwyu/bitcoin.core.imp new file mode 100644 index 0000000000..ce7786f58c --- /dev/null +++ b/contrib/devtools/iwyu/bitcoin.core.imp @@ -0,0 +1,6 @@ +# Fixups / upstreamed changes +[ + { include: [ "<bits/termios-c_lflag.h>", private, "<termios.h>", public ] }, + { include: [ "<bits/termios-struct.h>", private, "<termios.h>", public ] }, + { include: [ "<bits/termios-tcflow.h>", private, "<termios.h>", public ] }, +] diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index c8f2e40f0a..74b24b9612 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -132,7 +132,7 @@ for host in $HOSTS; do echo "Found macOS SDK at '${OSX_SDK}', using..." break else - echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..." exit 1 fi ;; diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index fa7d953ce3..64e36f22b9 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -58,7 +58,7 @@ previous stage) as the first argument. ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` -The `sha256sum` of the generated TAR.GZ archive should be `e7ca56bc8804d16624fad68be2e71647747d6629cacaaa3de5fbfa7f444e9eae`. +The `sha256sum` of the generated TAR.GZ archive should be `df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619`. ## Deterministic macOS DMG Notes diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index d70cc8613c..6efaaccb8e 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -95,7 +95,7 @@ def run(): tarinfo.uid, tarinfo.uname = 0, '' tarinfo.gid, tarinfo.gname = 0, '' # don't use isdir() as there are also executable files present - tarinfo.mode = 0o0755 if tarinfo.mode & 0o0100 else 0x0644 + tarinfo.mode = 0o0755 if tarinfo.mode & 0o0100 else 0o0644 return tarinfo with cd(dir_to_add): # recursion already adds entries in sorted order diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index 2f370d2b87..48020d71af 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -13,4 +13,4 @@ mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -mingw_cmake_system=Windows +mingw32_cmake_system=Windows diff --git a/doc/REST-interface.md b/doc/REST-interface.md index f11624fd28..4b46f29153 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -125,6 +125,7 @@ Refer to the `getmempoolinfo` RPC help for details. Returns the transactions in the mempool. Only supports JSON as output format. +Refer to the `getrawmempool` RPC help for details. Risks ------------- diff --git a/doc/release-notes/release-notes-23.0.md b/doc/release-notes/release-notes-23.0.md new file mode 100644 index 0000000000..b1467a0f71 --- /dev/null +++ b/doc/release-notes/release-notes-23.0.md @@ -0,0 +1,373 @@ +23.0 Release Notes +================== + +Bitcoin Core version 23.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-23.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +- A bitcoind node will no longer rumour addresses to inbound peers by default. + They will become eligible for address gossip after sending an ADDR, ADDRV2, + or GETADDR message. (#21528) + +- Before this release, Bitcoin Core had a strong preference to try to connect only to peers that listen on port 8333. As a result of that, Bitcoin nodes listening on non-standard ports would likely not get any Bitcoin Core peers connecting to them. This preference has been removed. (#23542) + +- Full support has been added for the CJDNS network. See the new option `-cjdnsreachable` and [doc/cjdns.md](https://github.com/bitcoin/bitcoin/tree/23.x/doc/cjdns.md) (#23077) + +Fee estimation changes +---------------------- + +- Fee estimation now takes the feerate of replacement (RBF) transactions into + account. (#22539) + +Rescan startup parameter removed +-------------------------------- + +The `-rescan` startup parameter has been removed. Wallets which require +rescanning due to corruption will still be rescanned on startup. +Otherwise, please use the `rescanblockchain` RPC to trigger a rescan. (#23123) + +Tracepoints and Userspace, Statically Defined Tracing support +------------------------------------------------------------- + +Bitcoin Core release binaries for Linux now include experimental tracepoints which +act as an interface for process-internal events. These can be used for review, +debugging, monitoring, and more. The tracepoint API is semi-stable. While the API +is tested, process internals might change between releases requiring changes to the +tracepoints. Information about the existing tracepoints can be found under +[doc/tracing.md](https://github.com/bitcoin/bitcoin/blob/23.x/doc/tracing.md) and +usage examples are provided in [contrib/tracing/](https://github.com/bitcoin/bitcoin/tree/23.x/contrib/tracing). + +Updated RPCs +------------ + +- The `validateaddress` RPC now returns an `error_locations` array for invalid + addresses, with the indices of invalid character locations in the address (if + known). For example, this will attempt to locate up to two Bech32 errors, and + return their locations if successful. Success and correctness are only guaranteed + if fewer than two substitution errors have been made. + The error message returned in the `error` field now also returns more specific + errors when decoding fails. (#16807) + +- The `-deprecatedrpc=addresses` configuration option has been removed. RPCs + `gettxout`, `getrawtransaction`, `decoderawtransaction`, `decodescript`, + `gettransaction verbose=true` and REST endpoints `/rest/tx`, `/rest/getutxos`, + `/rest/block` no longer return the `addresses` and `reqSigs` fields, which + were previously deprecated in 22.0. (#22650) +- The `getblock` RPC command now supports verbosity level 3 containing transaction inputs' + `prevout` information. The existing `/rest/block/` REST endpoint is modified to contain + this information too. Every `vin` field will contain an additional `prevout` subfield + describing the spent output. `prevout` contains the following keys: + - `generated` - true if the spent coins was a coinbase. + - `height` + - `value` + - `scriptPubKey` + +- The top-level fee fields `fee`, `modifiedfee`, `ancestorfees` and `descendantfees` + returned by RPCs `getmempoolentry`,`getrawmempool(verbose=true)`, + `getmempoolancestors(verbose=true)` and `getmempooldescendants(verbose=true)` + are deprecated and will be removed in the next major version (use + `-deprecated=fees` if needed in this version). The same fee fields can be accessed + through the `fees` object in the result. WARNING: deprecated + fields `ancestorfees` and `descendantfees` are denominated in sats, whereas all + fields in the `fees` object are denominated in BTC. (#22689) + +- Both `createmultisig` and `addmultisigaddress` now include a `warnings` + field, which will show a warning if a non-legacy address type is requested + when using uncompressed public keys. (#23113) + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +- Information on soft fork status has been moved from `getblockchaininfo` + to the new `getdeploymentinfo` RPC which allows querying soft fork status at any + block, rather than just at the chain tip. Inclusion of soft fork + status in `getblockchaininfo` can currently be restored using the + configuration `-deprecatedrpc=softforks`, but this will be removed in + a future release. Note that in either case, the `status` field + now reflects the status of the current block rather than the next + block. (#23508) + +Files +----- + +* On startup, the list of banned hosts and networks (via `setban` RPC) in + `banlist.dat` is ignored and only `banlist.json` is considered. Bitcoin Core + version 22.x is the only version that can read `banlist.dat` and also write + it to `banlist.json`. If `banlist.json` already exists, version 22.x will not + try to translate the `banlist.dat` into json. After an upgrade, `listbanned` + can be used to double check the parsed entries. (#22570) + +Updated settings +---------------- + +- In previous releases, the meaning of the command line option + `-persistmempool` (without a value provided) incorrectly disabled mempool + persistence. `-persistmempool` is now treated like other boolean options to + mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` + and `-nopersistmempool` is unaffected. (#23061) + +- `-maxuploadtarget` now allows human readable byte units [k|K|m|M|g|G|t|T]. + E.g. `-maxuploadtarget=500g`. No whitespace, +- or fractions allowed. + Default is `M` if no suffix provided. (#23249) + +- If `-proxy=` is given together with `-noonion` then the provided proxy will + not be set as a proxy for reaching the Tor network. So it will not be + possible to open manual connections to the Tor network for example with the + `addnode` RPC. To mimic the old behavior use `-proxy=` together with + `-onlynet=` listing all relevant networks except `onion`. (#22834) + +Tools and Utilities +------------------- + +- Update `-getinfo` to return data in a user-friendly format that also reduces vertical space. (#21832) + +- CLI `-addrinfo` now returns a single field for the number of `onion` addresses + known to the node instead of separate `torv2` and `torv3` fields, as support + for Tor V2 addresses was removed from Bitcoin Core in 22.0. (#22544) + +Wallet +------ + +- Descriptor wallets are now the default wallet type. Newly created wallets + will use descriptors unless `descriptors=false` is set during `createwallet`, or + the `Descriptor wallet` checkbox is unchecked in the GUI. + + Note that wallet RPC commands like `importmulti` and `dumpprivkey` cannot be + used with descriptor wallets, so if your client code relies on these commands + without specifying `descriptors=false` during wallet creation, you will need + to update your code. + +- Newly created descriptor wallets will contain an automatically generated `tr()` + descriptor which allows for creating single key Taproot receiving addresses. + +- `upgradewallet` will now automatically flush the keypool if upgrading + from a non-HD wallet to an HD wallet, to immediately start using the + newly-generated HD keys. (#23093) + +- a new RPC `newkeypool` has been added, which will flush (entirely + clear and refill) the keypool. (#23093) + +- `listunspent` now includes `ancestorcount`, `ancestorsize`, and + `ancestorfees` for each transaction output that is still in the mempool. + (#12677) + +- `lockunspent` now optionally takes a third parameter, `persistent`, which + causes the lock to be written persistently to the wallet database. This + allows UTXOs to remain locked even after node restarts or crashes. (#23065) + +- `receivedby` RPCs now include coinbase transactions. Previously, the + following wallet RPCs excluded coinbase transactions: `getreceivedbyaddress`, + `getreceivedbylabel`, `listreceivedbyaddress`, `listreceivedbylabel`. This + release changes this behaviour and returns results accounting for received + coins from coinbase outputs. The previous behaviour can be restored using the + configuration `-deprecatedrpc=exclude_coinbase`, but may be removed in a + future release. (#14707) + +- A new option in the same `receivedby` RPCs, `include_immature_coinbase` + (default=`false`), determines whether to account for immature coinbase + transactions. Immature coinbase transactions are coinbase transactions that + have 100 or fewer confirmations, and are not spendable. (#14707) + +GUI changes +----------- + +- UTXOs which are locked via the GUI are now stored persistently in the + wallet database, so are not lost on node shutdown or crash. (#23065) + +- The Bech32 checkbox has been replaced with a dropdown for all address types, including the new Bech32m (BIP-350) standard for Taproot enabled wallets. + +Low-level changes +================= + +RPC +--- + +- `getblockchaininfo` now returns a new `time` field, that provides the chain tip time. (#22407) + +Tests +----- + +- For the `regtest` network the activation heights of several softforks were + set to block height 1. They can be changed by the runtime setting + `-testactivationheight=name@height`. (#22818) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- 0xree +- Aaron Clauson +- Adrian-Stefan Mares +- agroce +- aitorjs +- Alex Groce +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Poelstra +- Andrew Toth +- anouar kappitou +- Anthony Towns +- Antoine Poinsot +- Arnab Sen +- Ben Woosley +- benthecarman +- Bitcoin Hodler +- BitcoinTsunami +- brianddk +- Bruno Garcia +- CallMeMisterOwl +- Calvin Kim +- Carl Dong +- Cory Fields +- Cuong V. Nguyen +- Darius Parvin +- Dhruv Mehta +- Dimitri Deijs +- Dimitris Apostolou +- Dmitry Goncharov +- Douglas Chimento +- eugene +- Fabian Jahr +- fanquake +- Florian Baumgartl +- fyquah +- Gleb Naumenko +- glozow +- Gregory Sanders +- Heebs +- Hennadii Stepanov +- hg333 +- HiLivin +- Igor Cota +- Jadi +- James O'Beirne +- Jameson Lopp +- Jarol Rodriguez +- Jeremy Rand +- Jeremy Rubin +- Joan Karadimov +- John Newbery +- Jon Atack +- João Barbosa +- josibake +- junderw +- Karl-Johan Alm +- katesalazar +- Kennan Mell +- Kiminuo +- Kittywhiskers Van Gogh +- Klement Tan +- Kristaps Kaupe +- Kuro +- Larry Ruane +- lsilva01 +- lucash-dev +- Luke Dashjr +- MarcoFalke +- Martin Leitner-Ankerl +- Martin Zumsande +- Matt Corallo +- Matt Whitlock +- MeshCollider +- Michael Dietz +- Murch +- naiza +- Nathan Garabedian +- Nelson Galdeman +- NikhilBartwal +- Niklas Gögge +- node01 +- nthumann +- Pasta +- Patrick Kamin +- Pavel Safronov +- Pavol Rusnak +- Perlover +- Pieter Wuille +- practicalswift +- pradumnasaraf +- pranabp-bit +- Prateek Sancheti +- Prayank +- Rafael Sadowski +- rajarshimaitra +- randymcmillan +- ritickgoenka +- Rob Fielding +- Rojar Smith +- Russell Yanofsky +- S3RK +- Saibato +- Samuel Dobson +- sanket1729 +- seaona +- Sebastian Falbesoner +- sh15h4nk +- Shashwat +- Shorya +- ShubhamPalriwala +- Shubhankar Gambhir +- Sjors Provoost +- sogoagain +- sstone +- stratospher +- Suriyaa Rocky Sundararuban +- Taeik Lim +- TheCharlatan +- Tim Ruffing +- Tobin Harding +- Troy Giorshev +- Tyler Chambers +- Vasil Dimov +- W. J. van der Laan +- w0xlt +- willcl-ark +- William Casarin +- zealsham +- Zero-1729 + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/tracing.md b/doc/tracing.md index 6ec6a6c218..b6e3b9263a 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -168,6 +168,49 @@ Arguments passed: 4. Value of the coin as `int64` 5. If the coin is a coinbase as `bool` +### Context `coin_selection` + +#### Tracepoint `coin_selection:selected_coins` + +Is called when `SelectCoins` completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Coin selection algorithm name as `pointer to C-style string` +3. Selection target value as `int64` +4. Calculated waste metric of the solution as `int64` +5. Total value of the selected inputs as `int64` + +#### Tracepoint `coin_selection:normal_create_tx_internal` + +Is called when the first `CreateTransactionInternal` completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Whether `CreateTransactionInternal` succeeded as `bool` +3. The expected transaction fee as an `int64` +4. The position of the change output as an `int32` + +#### Tracepoint `coin_selection:attempting_aps_create_tx` + +Is called when `CreateTransactionInternal` is called the second time for the optimistic +Avoid Partial Spends selection attempt. This is used to determine whether the next +tracepoints called are for the Avoid Partial Spends solution, or a different transaction. + +Arguments passed: +1. Wallet name as `pointer to C-style string` + +#### Tracepoint `coin_selection:aps_create_tx_internal` + +Is called when the second `CreateTransactionInternal` with Avoid Partial Spends enabled completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Whether the Avoid Partial Spends solution will be used as `bool` +3. Whether `CreateTransactionInternal` succeeded as` bool` +4. The expected transaction fee as an `int64` +5. The position of the change output as an `int32` + ## Adding tracepoints to Bitcoin Core To add a new tracepoint, `#include <util/trace.h>` in the compilation unit where diff --git a/src/.bear-tidy-config b/src/.bear-tidy-config new file mode 100644 index 0000000000..111ef6ee44 --- /dev/null +++ b/src/.bear-tidy-config @@ -0,0 +1,15 @@ +{ + "output": { + "content": { + "include_only_existing_source": true, + "paths_to_include": [], + "paths_to_exclude": [ + "src/leveldb" + ] + }, + "format": { + "command_as_array": true, + "drop_output_field": false + } + } +} diff --git a/src/.clang-tidy b/src/.clang-tidy index 27616ad072..fda95967f5 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -1,2 +1,2 @@ -Checks: '-*,bugprone-argument-comment' -WarningsAsErrors: bugprone-argument-comment +Checks: '-*,bugprone-argument-comment,modernize-use-nullptr' +WarningsAsErrors: 'bugprone-argument-comment,modernize-use-nullptr' diff --git a/src/Makefile.am b/src/Makefile.am index c089bed0c9..f77c2f919c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -15,14 +15,23 @@ AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = +lib_LTLIBRARIES = +noinst_LTLIBRARIES = + +bin_PROGRAMS = +noinst_PROGRAMS = +TESTS = +BENCHMARKS = + BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) LIBBITCOIN_NODE=libbitcoin_node.a LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CONSENSUS=libbitcoin_consensus.a LIBBITCOIN_CLI=libbitcoin_cli.a +LIBBITCOIN_KERNEL=libbitcoin_kernel.a LIBBITCOIN_UTIL=libbitcoin_util.a -LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.a +LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.la LIBBITCOINQT=qt/libbitcoinqt.a LIBSECP256K1=secp256k1/libsecp256k1.la @@ -32,28 +41,32 @@ endif if BUILD_BITCOIN_LIBS LIBBITCOINCONSENSUS=libbitcoinconsensus.la endif +if BUILD_BITCOIN_KERNEL_LIB +LIBBITCOINKERNEL=libbitcoinkernel.la +endif if ENABLE_WALLET LIBBITCOIN_WALLET=libbitcoin_wallet.a LIBBITCOIN_WALLET_TOOL=libbitcoin_wallet_tool.a endif -LIBBITCOIN_CRYPTO= $(LIBBITCOIN_CRYPTO_BASE) +LIBBITCOIN_CRYPTO = $(LIBBITCOIN_CRYPTO_BASE) if ENABLE_SSE41 -LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.a +LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SSE41) endif if ENABLE_AVX2 -LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.a +LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2) endif if ENABLE_X86_SHANI -LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libbitcoin_crypto_x86_shani.a +LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libbitcoin_crypto_x86_shani.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_X86_SHANI) endif if ENABLE_ARM_SHANI -LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.a +LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_SHANI) endif +noinst_LTLIBRARIES += $(LIBBITCOIN_CRYPTO) $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(wildcard secp256k1/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) @@ -61,7 +74,6 @@ $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(w # Make is not made aware of per-object dependencies to avoid limiting building parallelization # But to build the less dependent modules first, we manually select their order here: EXTRA_LIBRARIES += \ - $(LIBBITCOIN_CRYPTO) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CONSENSUS) \ @@ -72,14 +84,6 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_WALLET_TOOL) \ $(LIBBITCOIN_ZMQ) -lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) -noinst_LTLIBRARIES = - -bin_PROGRAMS = -noinst_PROGRAMS = -TESTS = -BENCHMARKS = - if BUILD_BITCOIND bin_PROGRAMS += bitcoind endif @@ -180,6 +184,7 @@ BITCOIN_CORE_H = \ net_types.h \ netaddress.h \ netbase.h \ + netgroup.h \ netmessagemaker.h \ node/blockstorage.h \ node/caches.h \ @@ -352,6 +357,7 @@ libbitcoin_node_a_SOURCES = \ init.cpp \ mapport.cpp \ net.cpp \ + netgroup.cpp \ net_processing.cpp \ node/blockstorage.cpp \ node/caches.cpp \ @@ -462,9 +468,16 @@ libbitcoin_wallet_tool_a_SOURCES = \ $(BITCOIN_CORE_H) # crypto primitives library -crypto_libbitcoin_crypto_base_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_base_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_base_a_SOURCES = \ +crypto_libbitcoin_crypto_base_la_CPPFLAGS = $(AM_CPPFLAGS) + +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +crypto_libbitcoin_crypto_base_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_base_la_LDFLAGS = $(AM_LDFLAGS) -static + +crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ crypto/chacha_poly_aead.h \ @@ -496,32 +509,44 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/siphash.h if USE_ASM -crypto_libbitcoin_crypto_base_a_SOURCES += crypto/sha256_sse4.cpp +crypto_libbitcoin_crypto_base_la_SOURCES += crypto/sha256_sse4.cpp endif -crypto_libbitcoin_crypto_sse41_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_sse41_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_sse41_a_CXXFLAGS += $(SSE41_CXXFLAGS) -crypto_libbitcoin_crypto_sse41_a_CPPFLAGS += -DENABLE_SSE41 -crypto_libbitcoin_crypto_sse41_a_SOURCES = crypto/sha256_sse41.cpp - -crypto_libbitcoin_crypto_avx2_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_avx2_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_avx2_a_CXXFLAGS += $(AVX2_CXXFLAGS) -crypto_libbitcoin_crypto_avx2_a_CPPFLAGS += -DENABLE_AVX2 -crypto_libbitcoin_crypto_avx2_a_SOURCES = crypto/sha256_avx2.cpp - -crypto_libbitcoin_crypto_x86_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_x86_shani_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_x86_shani_a_CXXFLAGS += $(X86_SHANI_CXXFLAGS) -crypto_libbitcoin_crypto_x86_shani_a_CPPFLAGS += -DENABLE_X86_SHANI -crypto_libbitcoin_crypto_x86_shani_a_SOURCES = crypto/sha256_x86_shani.cpp - -crypto_libbitcoin_crypto_arm_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_arm_shani_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_arm_shani_a_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) -crypto_libbitcoin_crypto_arm_shani_a_CPPFLAGS += -DENABLE_ARM_SHANI -crypto_libbitcoin_crypto_arm_shani_a_SOURCES = crypto/sha256_arm_shani.cpp +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_sse41_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_sse41_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_sse41_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_sse41_la_CXXFLAGS += $(SSE41_CXXFLAGS) +crypto_libbitcoin_crypto_sse41_la_CPPFLAGS += -DENABLE_SSE41 +crypto_libbitcoin_crypto_sse41_la_SOURCES = crypto/sha256_sse41.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_avx2_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_avx2_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_avx2_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_avx2_la_CXXFLAGS += $(AVX2_CXXFLAGS) +crypto_libbitcoin_crypto_avx2_la_CPPFLAGS += -DENABLE_AVX2 +crypto_libbitcoin_crypto_avx2_la_SOURCES = crypto/sha256_avx2.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_x86_shani_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_x86_shani_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_x86_shani_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_x86_shani_la_CXXFLAGS += $(X86_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_x86_shani_la_CPPFLAGS += -DENABLE_X86_SHANI +crypto_libbitcoin_crypto_x86_shani_la_SOURCES = crypto/sha256_x86_shani.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_arm_shani_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS += -DENABLE_ARM_SHANI +crypto_libbitcoin_crypto_arm_shani_la_SOURCES = crypto/sha256_arm_shani.cpp # consensus: shared between all executables that validate any consensus rules. libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) @@ -681,7 +706,6 @@ bitcoin_bin_ldadd = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) @@ -780,8 +804,48 @@ bitcoin_util_LDADD = \ # # bitcoin-chainstate binary # -bitcoin_chainstate_SOURCES = \ - bitcoin-chainstate.cpp \ +bitcoin_chainstate_SOURCES = bitcoin-chainstate.cpp +bitcoin_chainstate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +bitcoin_chainstate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) + +# $(LIBTOOL_APP_LDFLAGS) deliberately omitted here so that we can test linking +# bitcoin-chainstate against libbitcoinkernel as a shared or static library by +# setting --{en,dis}able-shared. +bitcoin_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(PTHREAD_FLAGS) +bitcoin_chainstate_LDADD = $(LIBBITCOINKERNEL) +# + +# bitcoinkernel library # +if BUILD_BITCOIN_KERNEL_LIB +lib_LTLIBRARIES += $(LIBBITCOINKERNEL) + +libbitcoinkernel_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(PTHREAD_FLAGS) +libbitcoinkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) +libbitcoinkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) + +# libbitcoinkernel requires default symbol visibility, explicitly specify that +# here so that things still work even when user configures with +# --enable-reduce-exports +# +# Note this is a quick hack that will be removed as we incrementally define what +# to export from the library. +libbitcoinkernel_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -fvisibility=default + +# TODO: For now, Specify -static in both CXXFLAGS and LDFLAGS when building for +# windows targets so libtool will only build a static version of this +# library. There are unresolved problems when building dll's for mingw-w64 +# and attempting to statically embed libstdc++, libpthread, etc. +if TARGET_WINDOWS +libbitcoinkernel_la_LDFLAGS += -static +libbitcoinkernel_la_CXXFLAGS += -static +endif + +# TODO: libbitcoinkernel is a work in progress consensus engine library, as more +# and more modules are decoupled from the consensus engine, this list will +# shrink to only those which are absolutely necessary. For example, things +# like index/*.cpp will be removed. +libbitcoinkernel_la_SOURCES = \ + kernel/bitcoinkernel.cpp \ arith_uint256.cpp \ blockfilter.cpp \ chain.cpp \ @@ -860,26 +924,19 @@ bitcoin_chainstate_SOURCES = \ validationinterface.cpp \ versionbits.cpp \ warnings.cpp -bitcoin_chainstate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -bitcoin_chainstate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -bitcoin_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -bitcoin_chainstate_LDADD = \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBUNIVALUE) \ - $(LIBSECP256K1) \ - $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ - $(LIBMEMENV) # Required for obj/build.h to be generated first. # More details: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html -bitcoin_chainstate-clientversion.$(OBJEXT): obj/build.h +libbitcoinkernel_la-clientversion.l$(OBJEXT): obj/build.h +endif # BUILD_BITCOIN_KERNEL_LIB # # bitcoinconsensus library # if BUILD_BITCOIN_LIBS +lib_LTLIBRARIES += $(LIBBITCOINCONSENSUS) + include_HEADERS = script/bitcoinconsensus.h -libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) +libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_la_SOURCES) $(libbitcoin_consensus_a_SOURCES) libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 58a09cd4a4..e90c8530d8 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -52,15 +52,14 @@ nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bench_bench_bitcoin_LDADD = \ + $(LIBTEST_UTIL) \ $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ - $(LIBTEST_UTIL) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ $(LIBUNIVALUE) \ diff --git a/src/Makefile.crc32c.include b/src/Makefile.crc32c.include index 3cbe71792c..c4dd84991d 100644 --- a/src/Makefile.crc32c.include +++ b/src/Makefile.crc32c.include @@ -2,10 +2,9 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -LIBCRC32C_INT = crc32c/libcrc32c.a -LIBLEVELDB_SSE42_INT = leveldb/libleveldb_sse42.a +LIBCRC32C_INT = crc32c/libcrc32c.la -EXTRA_LIBRARIES += $(LIBCRC32C_INT) +noinst_LTLIBRARIES += $(LIBCRC32C_INT) LIBCRC32C = $(LIBCRC32C_INT) @@ -34,41 +33,49 @@ else CRC32C_CPPFLAGS_INT += -DBYTE_ORDER_BIG_ENDIAN=0 endif -crc32c_libcrc32c_a_CPPFLAGS = $(AM_CPPFLAGS) $(CRC32C_CPPFLAGS_INT) $(CRC32C_CPPFLAGS) -crc32c_libcrc32c_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) - -crc32c_libcrc32c_a_SOURCES = -crc32c_libcrc32c_a_SOURCES += crc32c/include/crc32c/crc32c.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64_check.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_internal.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_prefetch.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_read_le.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_round_up.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_sse42_check.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_sse42.h - -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c.cc -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_portable.cc +crc32c_libcrc32c_la_CPPFLAGS = $(AM_CPPFLAGS) $(CRC32C_CPPFLAGS_INT) $(CRC32C_CPPFLAGS) + +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +crc32c_libcrc32c_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crc32c_libcrc32c_la_LDFLAGS = $(AM_LDFLAGS) -static + +crc32c_libcrc32c_la_SOURCES = +crc32c_libcrc32c_la_SOURCES += crc32c/include/crc32c/crc32c.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_arm64.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_arm64_check.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_internal.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_prefetch.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_read_le.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_round_up.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_sse42_check.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_sse42.h + +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c.cc +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_portable.cc if ENABLE_SSE42 -LIBCRC32C_SSE42_INT = crc32c/libcrc32c_sse42.a -EXTRA_LIBRARIES += $(LIBCRC32C_SSE42_INT) +LIBCRC32C_SSE42_INT = crc32c/libcrc32c_sse42.la +noinst_LTLIBRARIES += $(LIBCRC32C_SSE42_INT) LIBCRC32C += $(LIBCRC32C_SSE42_INT) -crc32c_libcrc32c_sse42_a_CPPFLAGS = $(crc32c_libcrc32c_a_CPPFLAGS) -crc32c_libcrc32c_sse42_a_CXXFLAGS = $(crc32c_libcrc32c_a_CXXFLAGS) $(SSE42_CXXFLAGS) +crc32c_libcrc32c_sse42_la_CPPFLAGS = $(crc32c_libcrc32c_la_CPPFLAGS) +crc32c_libcrc32c_sse42_la_CXXFLAGS = $(crc32c_libcrc32c_la_CXXFLAGS) $(SSE42_CXXFLAGS) +crc32c_libcrc32c_sse42_la_LDFLAGS = $(crc32c_libcrc32c_la_LDFLAGS) -crc32c_libcrc32c_sse42_a_SOURCES = crc32c/src/crc32c_sse42.cc +crc32c_libcrc32c_sse42_la_SOURCES = crc32c/src/crc32c_sse42.cc endif if ENABLE_ARM_CRC -LIBCRC32C_ARM_CRC_INT = crc32c/libcrc32c_arm_crc.a -EXTRA_LIBRARIES += $(LIBCRC32C_ARM_CRC_INT) +LIBCRC32C_ARM_CRC_INT = crc32c/libcrc32c_arm_crc.la +noinst_LTLIBRARIES += $(LIBCRC32C_ARM_CRC_INT) LIBCRC32C += $(LIBCRC32C_ARM_CRC_INT) -crc32c_libcrc32c_arm_crc_a_CPPFLAGS = $(crc32c_libcrc32c_a_CPPFLAGS) -crc32c_libcrc32c_arm_crc_a_CXXFLAGS = $(crc32c_libcrc32c_a_CXXFLAGS) $(ARM_CRC_CXXFLAGS) +crc32c_libcrc32c_arm_crc_la_CPPFLAGS = $(crc32c_libcrc32c_la_CPPFLAGS) +crc32c_libcrc32c_arm_crc_la_CXXFLAGS = $(crc32c_libcrc32c_la_CXXFLAGS) $(ARM_CRC_CXXFLAGS) +crc32c_libcrc32c_arm_crc_la_LDFLAGS = $(crc32c_libcrc32c_la_LDFLAGS) -crc32c_libcrc32c_arm_crc_a_SOURCES = crc32c/src/crc32c_arm64.cc +crc32c_libcrc32c_arm_crc_la_SOURCES = crc32c/src/crc32c_arm64.cc endif diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index 3bec92482a..066f8940c5 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -2,11 +2,11 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -LIBLEVELDB_INT = leveldb/libleveldb.a -LIBMEMENV_INT = leveldb/libmemenv.a +LIBLEVELDB_INT = leveldb/libleveldb.la +LIBMEMENV_INT = leveldb/libmemenv.la -EXTRA_LIBRARIES += $(LIBLEVELDB_INT) -EXTRA_LIBRARIES += $(LIBMEMENV_INT) +noinst_LTLIBRARIES += $(LIBLEVELDB_INT) +noinst_LTLIBRARIES += $(LIBMEMENV_INT) LIBLEVELDB = $(LIBLEVELDB_INT) $(LIBCRC32C) LIBMEMENV = $(LIBMEMENV_INT) @@ -37,111 +37,118 @@ else LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX endif -leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) -leveldb_libleveldb_a_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized -Wsuggest-override -Werror=suggest-override, $(AM_CXXFLAGS)) $(PIE_FLAGS) +leveldb_libleveldb_la_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) -leveldb_libleveldb_a_SOURCES= -leveldb_libleveldb_a_SOURCES += leveldb/port/port_stdcxx.h -leveldb_libleveldb_a_SOURCES += leveldb/port/port.h -leveldb_libleveldb_a_SOURCES += leveldb/port/thread_annotations.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/db.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/options.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/comparator.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/filter_policy.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/slice.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table_builder.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/env.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/export.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/c.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/iterator.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/cache.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/dumpfile.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/write_batch.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/status.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_format.h -leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.h -leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.h -leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch_internal.h -leveldb_libleveldb_a_SOURCES += leveldb/db/filename.h -leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.h -leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.h -leveldb_libleveldb_a_SOURCES += leveldb/db/builder.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.h -leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.h -leveldb_libleveldb_a_SOURCES += leveldb/db/skiplist.h -leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.h -leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.h -leveldb_libleveldb_a_SOURCES += leveldb/db/snapshot.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.h -leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.h -leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.h -leveldb_libleveldb_a_SOURCES += leveldb/table/block.h -leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.h -leveldb_libleveldb_a_SOURCES += leveldb/table/merger.h -leveldb_libleveldb_a_SOURCES += leveldb/table/format.h -leveldb_libleveldb_a_SOURCES += leveldb/table/iterator_wrapper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.h -leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix_test_helper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/env_windows_test_helper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/arena.h -leveldb_libleveldb_a_SOURCES += leveldb/util/random.h -leveldb_libleveldb_a_SOURCES += leveldb/util/posix_logger.h -leveldb_libleveldb_a_SOURCES += leveldb/util/hash.h -leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.h -leveldb_libleveldb_a_SOURCES += leveldb/util/coding.h -leveldb_libleveldb_a_SOURCES += leveldb/util/testutil.h -leveldb_libleveldb_a_SOURCES += leveldb/util/mutexlock.h -leveldb_libleveldb_a_SOURCES += leveldb/util/logging.h -leveldb_libleveldb_a_SOURCES += leveldb/util/no_destructor.h -leveldb_libleveldb_a_SOURCES += leveldb/util/testharness.h -leveldb_libleveldb_a_SOURCES += leveldb/util/windows_logger.h +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +leveldb_libleveldb_la_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized -Wsuggest-override -Werror=suggest-override, $(AM_CXXFLAGS)) $(PIE_FLAGS) -static +leveldb_libleveldb_la_LDFLAGS = $(AM_LDFLAGS) -static -leveldb_libleveldb_a_SOURCES += leveldb/db/builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/c.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/dumpfile.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/filename.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/repair.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/block.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/format.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/iterator.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/merger.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/table_builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/table.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/arena.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/bloom.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/cache.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/coding.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/comparator.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/env.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/filter_policy.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/hash.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/logging.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/options.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/status.cc +leveldb_libleveldb_la_SOURCES= +leveldb_libleveldb_la_SOURCES += leveldb/port/port_stdcxx.h +leveldb_libleveldb_la_SOURCES += leveldb/port/port.h +leveldb_libleveldb_la_SOURCES += leveldb/port/thread_annotations.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/db.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/options.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/comparator.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/filter_policy.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/slice.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/table_builder.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/env.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/export.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/c.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/iterator.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/cache.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/dumpfile.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/table.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/write_batch.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/status.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_format.h +leveldb_libleveldb_la_SOURCES += leveldb/db/memtable.h +leveldb_libleveldb_la_SOURCES += leveldb/db/version_set.h +leveldb_libleveldb_la_SOURCES += leveldb/db/write_batch_internal.h +leveldb_libleveldb_la_SOURCES += leveldb/db/filename.h +leveldb_libleveldb_la_SOURCES += leveldb/db/version_edit.h +leveldb_libleveldb_la_SOURCES += leveldb/db/dbformat.h +leveldb_libleveldb_la_SOURCES += leveldb/db/builder.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_writer.h +leveldb_libleveldb_la_SOURCES += leveldb/db/db_iter.h +leveldb_libleveldb_la_SOURCES += leveldb/db/skiplist.h +leveldb_libleveldb_la_SOURCES += leveldb/db/db_impl.h +leveldb_libleveldb_la_SOURCES += leveldb/db/table_cache.h +leveldb_libleveldb_la_SOURCES += leveldb/db/snapshot.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_reader.h +leveldb_libleveldb_la_SOURCES += leveldb/table/filter_block.h +leveldb_libleveldb_la_SOURCES += leveldb/table/block_builder.h +leveldb_libleveldb_la_SOURCES += leveldb/table/block.h +leveldb_libleveldb_la_SOURCES += leveldb/table/two_level_iterator.h +leveldb_libleveldb_la_SOURCES += leveldb/table/merger.h +leveldb_libleveldb_la_SOURCES += leveldb/table/format.h +leveldb_libleveldb_la_SOURCES += leveldb/table/iterator_wrapper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/crc32c.h +leveldb_libleveldb_la_SOURCES += leveldb/util/env_posix_test_helper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/env_windows_test_helper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/arena.h +leveldb_libleveldb_la_SOURCES += leveldb/util/random.h +leveldb_libleveldb_la_SOURCES += leveldb/util/posix_logger.h +leveldb_libleveldb_la_SOURCES += leveldb/util/hash.h +leveldb_libleveldb_la_SOURCES += leveldb/util/histogram.h +leveldb_libleveldb_la_SOURCES += leveldb/util/coding.h +leveldb_libleveldb_la_SOURCES += leveldb/util/testutil.h +leveldb_libleveldb_la_SOURCES += leveldb/util/mutexlock.h +leveldb_libleveldb_la_SOURCES += leveldb/util/logging.h +leveldb_libleveldb_la_SOURCES += leveldb/util/no_destructor.h +leveldb_libleveldb_la_SOURCES += leveldb/util/testharness.h +leveldb_libleveldb_la_SOURCES += leveldb/util/windows_logger.h + +leveldb_libleveldb_la_SOURCES += leveldb/db/builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/c.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/dbformat.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/db_impl.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/db_iter.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/dumpfile.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/filename.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/log_reader.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/log_writer.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/memtable.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/repair.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/table_cache.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/version_edit.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/version_set.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/write_batch.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/block_builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/block.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/filter_block.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/format.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/iterator.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/merger.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/table_builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/table.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/two_level_iterator.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/arena.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/bloom.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/cache.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/coding.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/comparator.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/crc32c.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/filter_policy.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/hash.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/histogram.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/logging.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/options.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/status.cc if TARGET_WINDOWS -leveldb_libleveldb_a_SOURCES += leveldb/util/env_windows.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env_windows.cc else -leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env_posix.cc endif -leveldb_libmemenv_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS) -leveldb_libmemenv_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS) -leveldb_libmemenv_a_SOURCES = leveldb/helpers/memenv/memenv.cc -leveldb_libmemenv_a_SOURCES += leveldb/helpers/memenv/memenv.h +leveldb_libmemenv_la_CPPFLAGS = $(leveldb_libleveldb_la_CPPFLAGS) +leveldb_libmemenv_la_CXXFLAGS = $(leveldb_libleveldb_la_CXXFLAGS) +leveldb_libmemenv_la_LDFLAGS = $(leveldb_libleveldb_la_LDFLAGS) +leveldb_libmemenv_la_SOURCES = leveldb/helpers/memenv/memenv.cc +leveldb_libmemenv_la_SOURCES += leveldb/helpers/memenv/memenv.h diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 3491f07ee0..72037b3db2 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -330,7 +330,7 @@ endif if ENABLE_ZMQ bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ +bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) bitcoin_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 29c322fbc2..fa822f2954 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -55,7 +55,7 @@ if ENABLE_ZMQ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(QT_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) \ + $(LIBMEMENV) $(QT_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) \ $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) qt_test_test_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 96a9a74802..02a3f9ae7d 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -47,7 +47,6 @@ FUZZ_SUITE_LD_COMMON = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ $(MINISKETCH_LIBS) \ @@ -110,6 +109,7 @@ BITCOIN_TESTS =\ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ + test/orphanage_tests.cpp \ test/pmt_tests.cpp \ test/policy_fee_tests.cpp \ test/policyestimator_tests.cpp \ @@ -204,7 +204,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) endif test_test_bitcoin_LDADD += $(LIBBITCOIN_NODE) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ - $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) + $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 9574454fd2..8922dda3ad 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -18,8 +18,3 @@ libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ test/fuzz/util.cpp \ $(TEST_FUZZ_H) - -LIBTEST_FUZZ += $(LIBBITCOIN_NODE) -LIBTEST_FUZZ += $(LIBBITCOIN_COMMON) -LIBTEST_FUZZ += $(LIBBITCOIN_UTIL) -LIBTEST_FUZZ += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 92cb8a5ce6..9306bb6fcc 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -34,8 +34,3 @@ libtest_util_a_SOURCES = \ test/util/validation.cpp \ test/util/wallet.cpp \ $(TEST_UTIL_H) - -LIBTEST_UTIL += $(LIBBITCOIN_NODE) -LIBTEST_UTIL += $(LIBBITCOIN_COMMON) -LIBTEST_UTIL += $(LIBBITCOIN_UTIL) -LIBTEST_UTIL += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 1fa2644647..0a76f83150 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -13,6 +13,7 @@ #include <hash.h> #include <logging/timer.h> #include <netbase.h> +#include <netgroup.h> #include <random.h> #include <streams.h> #include <tinyformat.h> @@ -49,7 +50,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data { // Generate random temporary filename uint16_t randv = 0; - GetRandBytes((unsigned char*)&randv, sizeof(randv)); + GetRandBytes({(unsigned char*)&randv, sizeof(randv)}); std::string tmpfn = strprintf("%s.%04x", prefix, randv); // open temp output file, and associate with CAutoFile @@ -182,10 +183,10 @@ void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) DeserializeDB(ssPeers, addr, false); } -std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) +std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) { auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique<AddrMan>(asmap, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); int64_t nStart = GetTimeMillis(); const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -194,7 +195,7 @@ std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const A LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), GetTimeMillis() - nStart); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<AddrMan>(asmap, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { @@ -203,7 +204,7 @@ std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const A return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); } // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<AddrMan>(asmap, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { diff --git a/src/addrdb.h b/src/addrdb.h index 4bdafb64e4..627ef3ac3c 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -17,6 +17,7 @@ class ArgsManager; class AddrMan; class CAddress; class CDataStream; +class NetGroupManager; struct bilingual_str; bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); @@ -48,7 +49,7 @@ public: }; /** Returns an error string on failure */ -std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman); +std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman); /** * Dump the anchor IP address database (anchors.dat) diff --git a/src/addrman.cpp b/src/addrman.cpp index 2a08d99eef..f74729d47b 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -43,17 +43,17 @@ static constexpr size_t ADDRMAN_SET_TRIED_COLLISION_SIZE{10}; /** The maximum time we'll spend trying to resolve a tried table collision, in seconds */ static constexpr int64_t ADDRMAN_TEST_WINDOW{40*60}; // 40 minutes -int AddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool>& asmap) const +int AddrInfo::GetTriedBucket(const uint256& nKey, const NetGroupManager& netgroupman) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); - uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); + uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << netgroupman.GetGroup(*this) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; } -int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool>& asmap) const +int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const NetGroupManager& netgroupman) const { - std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(asmap); - uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); + std::vector<unsigned char> vchSourceGroupKey = netgroupman.GetGroup(src); + uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << netgroupman.GetGroup(*this) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); return hash2 % ADDRMAN_NEW_BUCKET_COUNT; } @@ -99,11 +99,11 @@ double AddrInfo::GetChance(int64_t nNow) const return fChance; } -AddrManImpl::AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio) +AddrManImpl::AddrManImpl(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio) : insecure_rand{deterministic} , nKey{deterministic ? uint256{1} : insecure_rand.rand256()} , m_consistency_check_ratio{consistency_check_ratio} - , m_asmap{std::move(asmap)} + , m_netgroupman{netgroupman} { for (auto& bucket : vvNew) { for (auto& entry : bucket) { @@ -218,11 +218,7 @@ void AddrManImpl::Serialize(Stream& s_) const } // Store asmap checksum after bucket entries so that it // can be ignored by older clients for backward compatibility. - uint256 asmap_checksum; - if (m_asmap.size() != 0) { - asmap_checksum = SerializeHash(m_asmap); - } - s << asmap_checksum; + s << m_netgroupman.GetAsmapChecksum(); } template <typename Stream> @@ -298,7 +294,7 @@ void AddrManImpl::Unserialize(Stream& s_) for (int n = 0; n < nTried; n++) { AddrInfo info; s >> info; - int nKBucket = info.GetTriedBucket(nKey, m_asmap); + int nKBucket = info.GetTriedBucket(nKey, m_netgroupman); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); if (info.IsValid() && vvTried[nKBucket][nKBucketPos] == -1) { @@ -335,10 +331,7 @@ void AddrManImpl::Unserialize(Stream& s_) // If the bucket count and asmap checksum haven't changed, then attempt // to restore the entries to the buckets/positions they were in before // serialization. - uint256 supplied_asmap_checksum; - if (m_asmap.size() != 0) { - supplied_asmap_checksum = SerializeHash(m_asmap); - } + uint256 supplied_asmap_checksum{m_netgroupman.GetAsmapChecksum()}; uint256 serialized_asmap_checksum; if (format >= Format::V2_ASMAP) { s >> serialized_asmap_checksum; @@ -371,7 +364,7 @@ void AddrManImpl::Unserialize(Stream& s_) } else { // In case the new table data cannot be used (bucket count wrong or new asmap), // try to give them a reference based on their primary source address. - bucket = info.GetNewBucket(nKey, m_asmap); + bucket = info.GetNewBucket(nKey, m_netgroupman); bucket_position = info.GetBucketPosition(nKey, true, bucket); if (vvNew[bucket][bucket_position] == -1) { vvNew[bucket][bucket_position] = entry_index; @@ -495,7 +488,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) AssertLockHeld(cs); // remove the entry from all new buckets - const int start_bucket{info.GetNewBucket(nKey, m_asmap)}; + const int start_bucket{info.GetNewBucket(nKey, m_netgroupman)}; for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; ++n) { const int bucket{(start_bucket + n) % ADDRMAN_NEW_BUCKET_COUNT}; const int pos{info.GetBucketPosition(nKey, true, bucket)}; @@ -510,7 +503,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) assert(info.nRefCount == 0); // which tried bucket to move the entry to - int nKBucket = info.GetTriedBucket(nKey, m_asmap); + int nKBucket = info.GetTriedBucket(nKey, m_netgroupman); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). @@ -526,7 +519,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) nTried--; // find which new bucket it belongs to - int nUBucket = infoOld.GetNewBucket(nKey, m_asmap); + int nUBucket = infoOld.GetNewBucket(nKey, m_netgroupman); int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket); ClearNew(nUBucket, nUBucketPos); assert(vvNew[nUBucket][nUBucketPos] == -1); @@ -594,7 +587,7 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ nNew++; } - int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); + int nUBucket = pinfo->GetNewBucket(nKey, source, m_netgroupman); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (vvNew[nUBucket][nUBucketPos] != nId) { @@ -610,7 +603,7 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", - addr.ToString(), addr.GetMappedAS(m_asmap), nUBucket, nUBucketPos); + addr.ToString(), m_netgroupman.GetMappedAS(addr), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); @@ -650,7 +643,7 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey, m_asmap); + int tried_bucket = info.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); // Will moving this address into tried evict another entry? @@ -669,7 +662,7 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT // move nId to the tried tables MakeTried(info, nId); LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", - addr.ToString(), addr.GetMappedAS(m_asmap), tried_bucket, tried_bucket_pos); + addr.ToString(), m_netgroupman.GetMappedAS(addr), tried_bucket, tried_bucket_pos); return true; } } @@ -863,7 +856,7 @@ void AddrManImpl::ResolveCollisions_() AddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. - int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); + int tried_bucket = info_new.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket); if (!info_new.IsValid()) { // id_new may no longer map to a valid address erase_collision = true; @@ -929,7 +922,7 @@ std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision_() const AddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to - int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); + int tried_bucket = newInfo.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); const AddrInfo& info_old = mapInfo[vvTried[tried_bucket][tried_bucket_pos]]; @@ -945,13 +938,13 @@ std::optional<AddressPosition> AddrManImpl::FindAddressEntry_(const CAddress& ad if (!addr_info) return std::nullopt; if(addr_info->fInTried) { - int bucket{addr_info->GetTriedBucket(nKey, m_asmap)}; + int bucket{addr_info->GetTriedBucket(nKey, m_netgroupman)}; return AddressPosition(/*tried_in=*/true, /*multiplicity_in=*/1, /*bucket_in=*/bucket, /*position_in=*/addr_info->GetBucketPosition(nKey, false, bucket)); } else { - int bucket{addr_info->GetNewBucket(nKey, m_asmap)}; + int bucket{addr_info->GetNewBucket(nKey, m_netgroupman)}; return AddressPosition(/*tried_in=*/false, /*multiplicity_in=*/addr_info->nRefCount, /*bucket_in=*/bucket, @@ -1026,7 +1019,7 @@ int AddrManImpl::CheckAddrman() const if (!setTried.count(vvTried[n][i])) return -11; const auto it{mapInfo.find(vvTried[n][i])}; - if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_asmap) != n) { + if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_netgroupman) != n) { return -17; } if (it->second.GetBucketPosition(nKey, false, n) != i) { @@ -1154,13 +1147,8 @@ std::optional<AddressPosition> AddrManImpl::FindAddressEntry(const CAddress& add return entry; } -const std::vector<bool>& AddrManImpl::GetAsmap() const -{ - return m_asmap; -} - -AddrMan::AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio) - : m_impl(std::make_unique<AddrManImpl>(std::move(asmap), deterministic, consistency_check_ratio)) {} +AddrMan::AddrMan(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio) + : m_impl(std::make_unique<AddrManImpl>(netgroupman, deterministic, consistency_check_ratio)) {} AddrMan::~AddrMan() = default; @@ -1235,11 +1223,6 @@ void AddrMan::SetServices(const CService& addr, ServiceFlags nServices) m_impl->SetServices(addr, nServices); } -const std::vector<bool>& AddrMan::GetAsmap() const -{ - return m_impl->GetAsmap(); -} - std::optional<AddressPosition> AddrMan::FindAddressEntry(const CAddress& addr) { return m_impl->FindAddressEntry(addr); diff --git a/src/addrman.h b/src/addrman.h index 472282833b..a0063e8a9c 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -7,6 +7,7 @@ #define BITCOIN_ADDRMAN_H #include <netaddress.h> +#include <netgroup.h> #include <protocol.h> #include <streams.h> #include <timedata.h> @@ -88,7 +89,7 @@ protected: const std::unique_ptr<AddrManImpl> m_impl; public: - explicit AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio); + explicit AddrMan(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio); ~AddrMan(); @@ -172,8 +173,6 @@ public: //! Update an entry's service bits. void SetServices(const CService& addr, ServiceFlags nServices); - const std::vector<bool>& GetAsmap() const; - /** Test-only function * Find the address record in AddrMan and return information about its * position. diff --git a/src/addrman_impl.h b/src/addrman_impl.h index 5e76f72342..9d98cdde54 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -76,15 +76,15 @@ public: } //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; + int GetTriedBucket(const uint256& nKey, const NetGroupManager& netgroupman) const; //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; + int GetNewBucket(const uint256& nKey, const CNetAddr& src, const NetGroupManager& netgroupman) const; //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const + int GetNewBucket(const uint256& nKey, const NetGroupManager& netgroupman) const { - return GetNewBucket(nKey, source, asmap); + return GetNewBucket(nKey, source, netgroupman); } //! Calculate in which position of a bucket to store this entry. @@ -100,7 +100,7 @@ public: class AddrManImpl { public: - AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio); + AddrManImpl(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio); ~AddrManImpl(); @@ -140,8 +140,6 @@ public: std::optional<AddressPosition> FindAddressEntry(const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(!cs); - const std::vector<bool>& GetAsmap() const; - friend class AddrManDeterministic; private: @@ -212,21 +210,8 @@ private: /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ const int32_t m_consistency_check_ratio; - // Compressed IP->ASN mapping, loaded from a file when a node starts. - // Should be always empty if no file was provided. - // This mapping is then used for bucketing nodes in Addrman. - // - // If asmap is provided, nodes will be bucketed by - // AS they belong to, in order to make impossible for a node - // to connect to several nodes hosted in a single AS. - // This is done in response to Erebus attack, but also to generally - // diversify the connections every node creates, - // especially useful when a large fraction of nodes - // operate under a couple of cloud providers. - // - // If a new asmap was provided, the existing records - // would be re-bucketed accordingly. - const std::vector<bool> m_asmap; + /** Reference to the netgroup manager. netgroupman must be constructed before addrman and destructed after. */ + const NetGroupManager& m_netgroupman; //! Find an entry. AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/arith_uint256.h b/src/arith_uint256.h index 19193972a4..b7b3b3a285 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -24,22 +24,19 @@ template<unsigned int BITS> class base_uint { protected: + static_assert(BITS / 32 > 0 && BITS % 32 == 0, "Template parameter BITS must be a positive multiple of 32."); static constexpr int WIDTH = BITS / 32; uint32_t pn[WIDTH]; public: base_uint() { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - for (int i = 0; i < WIDTH; i++) pn[i] = 0; } base_uint(const base_uint& b) { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - for (int i = 0; i < WIDTH; i++) pn[i] = b.pn[i]; } @@ -53,8 +50,6 @@ public: base_uint(uint64_t b) { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - pn[0] = (unsigned int)b; pn[1] = (unsigned int)(b >> 32); for (int i = 2; i < WIDTH; i++) diff --git a/src/base58.cpp b/src/base58.cpp index dfa2e8db55..11c1ce7397 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -126,7 +126,7 @@ std::string EncodeBase58(Span<const unsigned char> input) bool DecodeBase58(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret_len) { - if (!ValidAsCString(str)) { + if (!ContainsNoNUL(str)) { return false; } return DecodeBase58(str.c_str(), vchRet, max_ret_len); @@ -160,7 +160,7 @@ std::string EncodeBase58Check(Span<const unsigned char> input) bool DecodeBase58Check(const std::string& str, std::vector<unsigned char>& vchRet, int max_ret) { - if (!ValidAsCString(str)) { + if (!ContainsNoNUL(str)) { return false; } return DecodeBase58Check(str.c_str(), vchRet, max_ret); diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 34bc4380dd..76300f4db8 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -4,6 +4,7 @@ #include <addrman.h> #include <bench/bench.h> +#include <netgroup.h> #include <random.h> #include <util/check.h> #include <util/time.h> @@ -16,7 +17,7 @@ static constexpr size_t NUM_SOURCES = 64; static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256; -static const std::vector<bool> EMPTY_ASMAP; +static NetGroupManager EMPTY_NETGROUPMAN{std::vector<bool>()}; static constexpr uint32_t ADDRMAN_CONSISTENCY_CHECK_RATIO{0}; static std::vector<CAddress> g_sources; @@ -77,14 +78,14 @@ static void AddrManAdd(benchmark::Bench& bench) CreateAddresses(); bench.run([&] { - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; AddAddressesToAddrMan(addrman); }); } static void AddrManSelect(benchmark::Bench& bench) { - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); @@ -96,7 +97,7 @@ static void AddrManSelect(benchmark::Bench& bench) static void AddrManGetAddr(benchmark::Bench& bench) { - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); @@ -125,7 +126,7 @@ static void AddrManAddThenGood(benchmark::Bench& bench) // // This has some overhead (exactly the result of AddrManAdd benchmark), but that overhead is constant so improvements in // AddrMan::Good() will still be noticeable. - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; AddAddressesToAddrMan(addrman); markSomeAsGood(addrman); diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index fcbb6aacce..e5b6875a36 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -28,8 +28,6 @@ #include <functional> #include <iosfwd> -const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; - int main(int argc, char* argv[]) { // SETUP: Argument parsing and handling @@ -256,7 +254,5 @@ epilogue: } GetMainSignals().UnregisterBackgroundSignalScheduler(); - WITH_LOCK(::cs_main, UnloadBlockIndex(nullptr, chainman)); - init::UnsetGlobals(); } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 0b40626595..cdc5960c12 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -31,8 +31,6 @@ #include <memory> #include <stdio.h> -#include <boost/algorithm/string.hpp> - static bool fCreateBlank; static std::map<std::string,UniValue> registers; static const int CONTINUE_EXECUTION=-1; @@ -242,7 +240,7 @@ static void MutateTxRBFOptIn(CMutableTransaction& tx, const std::string& strInId template <typename T> static T TrimAndParse(const std::string& int_str, const std::string& err) { - const auto parsed{ToIntegral<T>(TrimString(int_str))}; + const auto parsed{ToIntegral<T>(TrimStringView(int_str))}; if (!parsed.has_value()) { throw std::runtime_error(err + " '" + int_str + "'"); } @@ -251,8 +249,7 @@ static T TrimAndParse(const std::string& int_str, const std::string& err) static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInput) { - std::vector<std::string> vStrInputParts; - boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + std::vector<std::string> vStrInputParts = SplitString(strInput, ':'); // separate TXID:VOUT in string if (vStrInputParts.size()<2) @@ -287,8 +284,7 @@ static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInpu static void MutateTxAddOutAddr(CMutableTransaction& tx, const std::string& strInput) { // Separate into VALUE:ADDRESS - std::vector<std::string> vStrInputParts; - boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + std::vector<std::string> vStrInputParts = SplitString(strInput, ':'); if (vStrInputParts.size() != 2) throw std::runtime_error("TX output missing or too many separators"); @@ -312,8 +308,7 @@ static void MutateTxAddOutAddr(CMutableTransaction& tx, const std::string& strIn static void MutateTxAddOutPubKey(CMutableTransaction& tx, const std::string& strInput) { // Separate into VALUE:PUBKEY[:FLAGS] - std::vector<std::string> vStrInputParts; - boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + std::vector<std::string> vStrInputParts = SplitString(strInput, ':'); if (vStrInputParts.size() < 2 || vStrInputParts.size() > 3) throw std::runtime_error("TX output missing or too many separators"); @@ -356,8 +351,7 @@ static void MutateTxAddOutPubKey(CMutableTransaction& tx, const std::string& str static void MutateTxAddOutMultiSig(CMutableTransaction& tx, const std::string& strInput) { // Separate into VALUE:REQUIRED:NUMKEYS:PUBKEY1:PUBKEY2:....[:FLAGS] - std::vector<std::string> vStrInputParts; - boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + std::vector<std::string> vStrInputParts = SplitString(strInput, ':'); // Check that there are enough parameters if (vStrInputParts.size()<3) @@ -460,8 +454,7 @@ static void MutateTxAddOutData(CMutableTransaction& tx, const std::string& strIn static void MutateTxAddOutScript(CMutableTransaction& tx, const std::string& strInput) { // separate VALUE:SCRIPT[:FLAGS] - std::vector<std::string> vStrInputParts; - boost::split(vStrInputParts, strInput, boost::is_any_of(":")); + std::vector<std::string> vStrInputParts = SplitString(strInput, ':'); if (vStrInputParts.size() < 2) throw std::runtime_error("TX output missing separator"); diff --git a/src/chainparams.cpp b/src/chainparams.cpp index c65a816a5f..7a7c72ea25 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -10,13 +10,11 @@ #include <deploymentinfo.h> #include <hash.h> // for signet block challenge hash #include <script/interpreter.h> +#include <util/string.h> #include <util/system.h> #include <assert.h> -#include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/split.hpp> - static CBlock CreateGenesisBlock(const char* pszTimestamp, const CScript& genesisOutputScript, uint32_t nTime, uint32_t nNonce, uint32_t nBits, int32_t nVersion, const CAmount& genesisReward) { CMutableTransaction txNew; @@ -528,8 +526,7 @@ void CRegTestParams::UpdateActivationParametersFromArgs(const ArgsManager& args) if (!args.IsArgSet("-vbparams")) return; for (const std::string& strDeployment : args.GetArgs("-vbparams")) { - std::vector<std::string> vDeploymentParams; - boost::split(vDeploymentParams, strDeployment, boost::is_any_of(":")); + std::vector<std::string> vDeploymentParams = SplitString(strDeployment, ':'); if (vDeploymentParams.size() < 3 || 4 < vDeploymentParams.size()) { throw std::runtime_error("Version bits parameters malformed, expecting deployment:start:end[:min_activation_height]"); } diff --git a/src/compat/byteswap.h b/src/compat/byteswap.h index 27ef1a18df..2f4232fa5c 100644 --- a/src/compat/byteswap.h +++ b/src/compat/byteswap.h @@ -9,7 +9,7 @@ #include <config/bitcoin-config.h> #endif -#include <stdint.h> +#include <cstdint> #if defined(HAVE_BYTESWAP_H) #include <byteswap.h> diff --git a/src/compat/cpuid.h b/src/compat/cpuid.h index 0877ad47d3..e78c1ce6d1 100644 --- a/src/compat/cpuid.h +++ b/src/compat/cpuid.h @@ -10,6 +10,8 @@ #include <cpuid.h> +#include <cstdint> + // We can't use cpuid.h's __get_cpuid as it does not support subleafs. void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) { diff --git a/src/compat/endian.h b/src/compat/endian.h index c5cf7a46cc..bdd8b84c1b 100644 --- a/src/compat/endian.h +++ b/src/compat/endian.h @@ -11,7 +11,7 @@ #include <compat/byteswap.h> -#include <stdint.h> +#include <cstdint> #if defined(HAVE_ENDIAN_H) #include <endian.h> diff --git a/src/compat/glibcxx_sanity.cpp b/src/compat/glibcxx_sanity.cpp index e6e6208e40..f2ceeeeb9c 100644 --- a/src/compat/glibcxx_sanity.cpp +++ b/src/compat/glibcxx_sanity.cpp @@ -5,6 +5,7 @@ #include <list> #include <locale> #include <stdexcept> +#include <string> namespace { diff --git a/src/compat/stdin.cpp b/src/compat/stdin.cpp index 0fc4e0fcf2..61d66e39f2 100644 --- a/src/compat/stdin.cpp +++ b/src/compat/stdin.cpp @@ -2,23 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif +#include <compat/stdin.h> -#include <cstdio> // for fileno(), stdin +#include <cstdio> #ifdef WIN32 -#include <windows.h> // for SetStdinEcho() -#include <io.h> // for isatty() +#include <windows.h> +#include <io.h> #else -#include <termios.h> // for SetStdinEcho() -#include <unistd.h> // for SetStdinEcho(), isatty() -#include <poll.h> // for StdinReady() +#include <termios.h> +#include <unistd.h> +#include <poll.h> #endif -#include <compat/stdin.h> - // https://stackoverflow.com/questions/1413445/reading-a-password-from-stdcin void SetStdinEcho(bool enable) { diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index b0ea80ea1a..50a601c684 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -227,7 +227,7 @@ const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8; std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const { std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES); - GetRandBytes(ret.data(), OBFUSCATE_KEY_NUM_BYTES); + GetRandBytes(ret); return ret; } diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 5d0b59f7cb..93d9acf5da 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -131,8 +131,11 @@ static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUserna return false; if (strAuth.substr(0, 6) != "Basic ") return false; - std::string strUserPass64 = TrimString(strAuth.substr(6)); - std::string strUserPass = DecodeBase64(strUserPass64); + std::string_view strUserPass64 = TrimStringView(std::string_view{strAuth}.substr(6)); + auto userpass_data = DecodeBase64(strUserPass64); + std::string strUserPass; + if (!userpass_data) return false; + strUserPass.assign(userpass_data->begin(), userpass_data->end()); if (strUserPass.find(':') != std::string::npos) strAuthUsernameOut = strUserPass.substr(0, strUserPass.find(':')); diff --git a/src/i2p.cpp b/src/i2p.cpp index ccba14d63d..e08b5461fe 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -69,12 +69,11 @@ static std::string SwapBase64(const std::string& from) static Binary DecodeI2PBase64(const std::string& i2p_b64) { const std::string& std_b64 = SwapBase64(i2p_b64); - bool invalid; - Binary decoded = DecodeBase64(std_b64.c_str(), &invalid); - if (invalid) { + auto decoded = DecodeBase64(std_b64); + if (!decoded) { throw std::runtime_error(strprintf("Cannot decode Base64: \"%s\"", i2p_b64)); } - return decoded; + return std::move(*decoded); } /** diff --git a/src/index/base.cpp b/src/index/base.cpp index 8fe30f8960..488a214ccf 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -65,9 +65,9 @@ bool BaseIndex::Init() LOCK(cs_main); CChain& active_chain = m_chainstate->m_chain; if (locator.IsNull()) { - m_best_block_index = nullptr; + SetBestBlockIndex(nullptr); } else { - m_best_block_index = m_chainstate->FindForkInGlobalIndex(locator); + SetBestBlockIndex(m_chainstate->FindForkInGlobalIndex(locator)); } m_synced = m_best_block_index.load() == active_chain.Tip(); if (!m_synced) { @@ -75,11 +75,7 @@ bool BaseIndex::Init() if (!m_best_block_index) { // index is not built yet // make sure we have all block data back to the genesis - const CBlockIndex* block = active_chain.Tip(); - while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { - block = block->pprev; - } - prune_violation = block != active_chain.Genesis(); + prune_violation = node::GetFirstStoredBlock(active_chain.Tip()) != active_chain.Genesis(); } // in case the index has a best block set and is not fully synced // check if we have the required blocks to continue building the index @@ -138,7 +134,7 @@ void BaseIndex::ThreadSync() int64_t last_locator_write_time = 0; while (true) { if (m_interrupt) { - m_best_block_index = pindex; + SetBestBlockIndex(pindex); // No need to handle errors in Commit. If it fails, the error will be already be // logged. The best way to recover is to continue, as index cannot be corrupted by // a missed commit to disk for an advanced index state. @@ -150,7 +146,7 @@ void BaseIndex::ThreadSync() LOCK(cs_main); const CBlockIndex* pindex_next = NextSyncBlock(pindex, m_chainstate->m_chain); if (!pindex_next) { - m_best_block_index = pindex; + SetBestBlockIndex(pindex); m_synced = true; // No need to handle errors in Commit. See rationale above. Commit(); @@ -172,7 +168,7 @@ void BaseIndex::ThreadSync() } if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { - m_best_block_index = pindex; + SetBestBlockIndex(pindex); last_locator_write_time = current_time; // No need to handle errors in Commit. See rationale above. Commit(); @@ -230,10 +226,10 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti // out of sync may be possible but a users fault. // In case we reorg beyond the pruned depth, ReadBlockFromDisk would // throw and lead to a graceful shutdown - m_best_block_index = new_tip; + SetBestBlockIndex(new_tip); if (!Commit()) { // If commit fails, revert the best block index to avoid corruption. - m_best_block_index = current_tip; + SetBestBlockIndex(current_tip); return false; } @@ -274,7 +270,7 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const } if (WriteBlock(*block, pindex)) { - m_best_block_index = pindex; + SetBestBlockIndex(pindex); } else { FatalError("%s: Failed to write block %s to index", __func__, pindex->GetBlockHash().ToString()); @@ -381,3 +377,14 @@ IndexSummary BaseIndex::GetSummary() const summary.best_block_height = m_best_block_index ? m_best_block_index.load()->nHeight : 0; return summary; } + +void BaseIndex::SetBestBlockIndex(const CBlockIndex* block) { + assert(!node::fPruneMode || AllowPrune()); + + m_best_block_index = block; + if (AllowPrune() && block) { + node::PruneLockInfo prune_lock; + prune_lock.height_first = block->nHeight; + WITH_LOCK(::cs_main, m_chainstate->m_blockman.UpdatePruneLock(GetName(), prune_lock)); + } +} diff --git a/src/index/base.h b/src/index/base.h index c4a8215bc4..a8f6a18c8d 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -75,6 +75,9 @@ private: /// to a chain reorganization), the index must halt until Commit succeeds or else it could end up /// getting corrupted. bool Commit(); + + virtual bool AllowPrune() const = 0; + protected: CChainState* m_chainstate{nullptr}; @@ -103,6 +106,9 @@ protected: /// Get the name of the index for display in logs. virtual const char* GetName() const = 0; + /// Update the internal best block index as well as the prune lock. + void SetBestBlockIndex(const CBlockIndex* block); + public: /// Destructor interrupts sync thread if running and blocks until it exits. virtual ~BaseIndex(); diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index a049019c02..b1836fe12f 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -38,6 +38,8 @@ private: /** cache of block hash to filter header, to avoid disk access when responding to getcfcheckpt. */ std::unordered_map<uint256, uint256, FilterHeaderHasher> m_headers_cache GUARDED_BY(m_cs_headers_cache); + bool AllowPrune() const override { return true; } + protected: bool Init() override; diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index 24190ac137..6f53bb74fb 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -36,6 +36,8 @@ private: bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); + bool AllowPrune() const override { return true; } + protected: bool Init() override; diff --git a/src/index/txindex.h b/src/index/txindex.h index 2bbc602631..ec339abaa1 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -20,6 +20,8 @@ protected: private: const std::unique_ptr<DB> m_db; + bool AllowPrune() const override { return false; } + protected: bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; diff --git a/src/init.cpp b/src/init.cpp index c0af2b136f..fdcb2b8ac7 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -33,6 +33,7 @@ #include <net_permissions.h> #include <net_processing.h> #include <netbase.h> +#include <netgroup.h> #include <node/blockstorage.h> #include <node/caches.h> #include <node/chainstate.h> @@ -72,6 +73,7 @@ #include <validationinterface.h> #include <walletinitinterface.h> +#include <algorithm> #include <condition_variable> #include <cstdint> #include <cstdio> @@ -239,6 +241,7 @@ void Shutdown(NodeContext& node) node.connman.reset(); node.banman.reset(); node.addrman.reset(); + node.netgroupman.reset(); if (node.mempool && node.mempool->IsLoaded() && node.args->GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(*node.mempool); @@ -419,11 +422,11 @@ void SetupServerArgs(ArgsManager& argsman) -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -coinstatsindex. " + argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk. This will also rebuild active optional indexes.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead. Deactivate all optional indexes before running this.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-settings=<file>", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #if HAVE_SYSTEM argsman.AddArg("-startupnotify=<cmd>", "Execute command on startup.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -856,8 +859,6 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb if (args.GetIntArg("-prune", 0)) { if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); - if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) - return InitError(_("Prune mode is incompatible with -coinstatsindex.")); if (args.GetBoolArg("-reindex-chainstate", false)) { return InitError(_("Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.")); } @@ -1031,6 +1032,19 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb return InitError(_("No proxy server specified. Use -proxy=<ip> or -proxy=<ip:port>.")); } + if (args.GetBoolArg("-reindex-chainstate", false)) { + // indexes that must be deactivated to prevent index corruption, see #24630 + if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { + return InitError(_("-reindex-chainstate option is not compatible with -coinstatsindex. Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); + } + if (g_enabled_filter_types.count(BlockFilterType::BASIC)) { + return InitError(_("-reindex-chainstate option is not compatible with -blockfilterindex. Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); + } + if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + return InitError(_("-reindex-chainstate option is not compatible with -txindex. Please temporarily disable txindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); + } + } + #if defined(USE_SYSCALL_SANDBOX) if (args.IsArgSet("-sandbox") && !args.IsArgNegated("-sandbox")) { const std::string sandbox_arg{args.GetArg("-sandbox", "")}; @@ -1228,8 +1242,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const bool ignores_incoming_txs{args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)}; { - // Initialize addrman - assert(!node.addrman); // Read asmap file if configured std::vector<bool> asmap; @@ -1253,8 +1265,14 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("Using /16 prefix for IP bucketing\n"); } + // Initialize netgroup manager + assert(!node.netgroupman); + node.netgroupman = std::make_unique<NetGroupManager>(std::move(asmap)); + + // Initialize addrman + assert(!node.addrman); uiInterface.InitMessage(_("Loading P2P addresses…").translated); - if (const auto error{LoadAddrman(asmap, args, node.addrman)}) { + if (const auto error{LoadAddrman(*node.netgroupman, args, node.addrman)}) { return InitError(*error); } } @@ -1262,26 +1280,15 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.banman); node.banman = std::make_unique<BanMan>(gArgs.GetDataDirNet() / "banlist", &uiInterface, args.GetIntArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); assert(!node.connman); - node.connman = std::make_unique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), *node.addrman, args.GetBoolArg("-networkactive", true)); + node.connman = std::make_unique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), + GetRand(std::numeric_limits<uint64_t>::max()), + *node.addrman, *node.netgroupman, args.GetBoolArg("-networkactive", true)); assert(!node.fee_estimator); // Don't initialize fee estimation with old data if we don't relay transactions, // as they would never get updated. if (!ignores_incoming_txs) node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(); - assert(!node.mempool); - int check_ratio = std::min<int>(std::max<int>(args.GetIntArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); - node.mempool = std::make_unique<CTxMemPool>(node.fee_estimator.get(), check_ratio); - - assert(!node.chainman); - node.chainman = std::make_unique<ChainstateManager>(); - ChainstateManager& chainman = *node.chainman; - - assert(!node.peerman); - node.peerman = PeerManager::make(chainparams, *node.connman, *node.addrman, node.banman.get(), - chainman, *node.mempool, ignores_incoming_txs); - RegisterValidationInterface(node.peerman.get()); - // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; for (const std::string& cmt : args.GetArgs("-uacomment")) { @@ -1410,8 +1417,16 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("* Using %.1f MiB for chain state database\n", cache_sizes.coins_db * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); - bool fLoaded = false; - while (!fLoaded && !ShutdownRequested()) { + assert(!node.mempool); + assert(!node.chainman); + const int mempool_check_ratio = std::clamp<int>(args.GetIntArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0, 1000000); + + for (bool fLoaded = false; !fLoaded && !ShutdownRequested();) { + node.mempool = std::make_unique<CTxMemPool>(node.fee_estimator.get(), mempool_check_ratio); + + node.chainman = std::make_unique<ChainstateManager>(); + ChainstateManager& chainman = *node.chainman; + const bool fReset = fReindex; bilingual_str strLoadError; @@ -1543,6 +1558,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return false; } + ChainstateManager& chainman = *Assert(node.chainman); + + assert(!node.peerman); + node.peerman = PeerManager::make(chainparams, *node.connman, *node.addrman, node.banman.get(), + chainman, *node.mempool, ignores_incoming_txs); + RegisterValidationInterface(node.peerman.get()); + // ********************************************************* Step 8: start indexers if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { if (const auto error{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}) { diff --git a/src/init/bitcoin-gui.cpp b/src/init/bitcoin-gui.cpp index e297b48718..2fa4add4e5 100644 --- a/src/init/bitcoin-gui.cpp +++ b/src/init/bitcoin-gui.cpp @@ -9,6 +9,7 @@ #include <interfaces/node.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/check.h> #include <util/system.h> #include <memory> diff --git a/src/init/bitcoin-node.cpp b/src/init/bitcoin-node.cpp index 511a872bc0..78bc3e5980 100644 --- a/src/init/bitcoin-node.cpp +++ b/src/init/bitcoin-node.cpp @@ -9,6 +9,7 @@ #include <interfaces/node.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/check.h> #include <util/system.h> #include <memory> diff --git a/src/init/bitcoin-qt.cpp b/src/init/bitcoin-qt.cpp index 37d4e426c5..bb3bb945d0 100644 --- a/src/init/bitcoin-qt.cpp +++ b/src/init/bitcoin-qt.cpp @@ -8,6 +8,7 @@ #include <interfaces/node.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/check.h> #include <util/system.h> #include <memory> diff --git a/src/init/bitcoin-wallet.cpp b/src/init/bitcoin-wallet.cpp index e9dcde72fe..c8d499da10 100644 --- a/src/init/bitcoin-wallet.cpp +++ b/src/init/bitcoin-wallet.cpp @@ -4,6 +4,8 @@ #include <interfaces/init.h> +#include <memory> + namespace interfaces { std::unique_ptr<Init> MakeWalletInit(int argc, char* argv[], int& exit_status) { diff --git a/src/init/bitcoind.cpp b/src/init/bitcoind.cpp index 2addff07c1..b473ebb805 100644 --- a/src/init/bitcoind.cpp +++ b/src/init/bitcoind.cpp @@ -8,6 +8,7 @@ #include <interfaces/node.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/check.h> #include <util/system.h> #include <memory> diff --git a/src/init/common.cpp b/src/init/common.cpp index 688471b35d..eac6732968 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -9,16 +9,21 @@ #include <clientversion.h> #include <compat/sanity.h> #include <crypto/sha256.h> +#include <fs.h> #include <key.h> #include <logging.h> #include <node/ui_interface.h> #include <pubkey.h> #include <random.h> +#include <tinyformat.h> #include <util/system.h> #include <util/time.h> #include <util/translation.h> +#include <algorithm> #include <memory> +#include <string> +#include <vector> static std::unique_ptr<ECCVerifyHandle> globalVerifyHandle; diff --git a/src/kernel/bitcoinkernel.cpp b/src/kernel/bitcoinkernel.cpp new file mode 100644 index 0000000000..bb101ba186 --- /dev/null +++ b/src/kernel/bitcoinkernel.cpp @@ -0,0 +1,10 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <functional> +#include <string> + +// Define G_TRANSLATION_FUN symbol in libbitcoinkernel library so users of the +// library aren't required to export this symbol +extern const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; diff --git a/src/key.cpp b/src/key.cpp index a54569b39e..d1d521f97d 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -159,7 +159,7 @@ bool CKey::Check(const unsigned char *vch) { void CKey::MakeNewKey(bool fCompressedIn) { do { - GetStrongRandBytes(keydata.data(), keydata.size()); + GetStrongRandBytes(keydata); } while (!Check(keydata.data())); fValid = true; fCompressed = fCompressedIn; @@ -244,7 +244,7 @@ bool CKey::VerifyPubKey(const CPubKey& pubkey) const { } unsigned char rnd[8]; std::string str = "Bitcoin key verification\n"; - GetRandBytes(rnd, sizeof(rnd)); + GetRandBytes(rnd); uint256 hash; CHash256().Write(MakeUCharSpan(str)).Write(rnd).Finalize(hash); std::vector<unsigned char> vchSig; @@ -397,7 +397,7 @@ void ECC_Start() { { // Pass in a random blinding seed to the secp256k1 context. std::vector<unsigned char, secure_allocator<unsigned char>> vseed(32); - GetRandBytes(vseed.data(), 32); + GetRandBytes(vseed); bool ret = secp256k1_context_randomize(ctx, vseed.data()); assert(ret); } diff --git a/src/mapport.cpp b/src/mapport.cpp index 42ca366089..235e6f904c 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -193,7 +193,7 @@ static bool ProcessUpnp() std::string strDesc = PACKAGE_NAME " " + FormatFullVersion(); do { - r = UPNP_AddPortMapping(urls.controlURL, data.first.servicetype, port.c_str(), port.c_str(), lanaddr, strDesc.c_str(), "TCP", 0, "0"); + r = UPNP_AddPortMapping(urls.controlURL, data.first.servicetype, port.c_str(), port.c_str(), lanaddr, strDesc.c_str(), "TCP", nullptr, "0"); if (r != UPNPCOMMAND_SUCCESS) { ret = false; @@ -206,7 +206,7 @@ static bool ProcessUpnp() } while (g_mapport_interrupt.sleep_for(PORT_MAPPING_REANNOUNCE_PERIOD)); g_mapport_interrupt.reset(); - r = UPNP_DeletePortMapping(urls.controlURL, data.first.servicetype, port.c_str(), "TCP", 0); + r = UPNP_DeletePortMapping(urls.controlURL, data.first.servicetype, port.c_str(), "TCP", nullptr); LogPrintf("UPNP_DeletePortMapping() returned: %d\n", r); freeUPNPDevlist(devlist); devlist = nullptr; FreeUPNPUrls(&urls); diff --git a/src/net.cpp b/src/net.cpp index 5a0c057c3e..aa70bed226 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1567,6 +1567,8 @@ void CConnman::SocketEvents(const std::vector<CNode*>& nodes, void CConnman::SocketHandler() { + AssertLockNotHeld(m_total_bytes_sent_mutex); + std::set<SOCKET> recv_set; std::set<SOCKET> send_set; std::set<SOCKET> error_set; @@ -1593,6 +1595,8 @@ void CConnman::SocketHandlerConnected(const std::vector<CNode*>& nodes, const std::set<SOCKET>& send_set, const std::set<SOCKET>& error_set) { + AssertLockNotHeld(m_total_bytes_sent_mutex); + for (CNode* pnode : nodes) { if (interruptNet) return; @@ -1694,6 +1698,8 @@ void CConnman::SocketHandlerListening(const std::set<SOCKET>& recv_set) void CConnman::ThreadSocketHandler() { + AssertLockNotHeld(m_total_bytes_sent_mutex); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET); while (!interruptNet) { @@ -2001,7 +2007,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) case ConnectionType::BLOCK_RELAY: case ConnectionType::ADDR_FETCH: case ConnectionType::FEELER: - setConnected.insert(pnode->addr.GetGroup(addrman.GetAsmap())); + setConnected.insert(m_netgroupman.GetGroup(pnode->addr)); } // no default case, so the compiler can warn about missing cases } } @@ -2075,7 +2081,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) m_anchors.pop_back(); if (!addr.IsValid() || IsLocal(addr) || !IsReachable(addr) || !HasAllDesirableServiceFlags(addr.nServices) || - setConnected.count(addr.GetGroup(addrman.GetAsmap()))) continue; + setConnected.count(m_netgroupman.GetGroup(addr))) continue; addrConnect = addr; LogPrint(BCLog::NET, "Trying to make an anchor connection to %s\n", addrConnect.ToString()); break; @@ -2116,7 +2122,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) } // Require outbound connections, other than feelers, to be to distinct network groups - if (!fFeeler && setConnected.count(addr.GetGroup(addrman.GetAsmap()))) { + if (!fFeeler && setConnected.count(m_netgroupman.GetGroup(addr))) { break; } @@ -2512,8 +2518,12 @@ void CConnman::SetNetworkActive(bool active) } } -CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, AddrMan& addrman_in, bool network_active) - : addrman(addrman_in), nSeed0(nSeed0In), nSeed1(nSeed1In) +CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, AddrMan& addrman_in, + const NetGroupManager& netgroupman, bool network_active) + : addrman(addrman_in) + , m_netgroupman{netgroupman} + , nSeed0(nSeed0In) + , nSeed1(nSeed1In) { SetTryNewOutboundPeer(false); @@ -2574,6 +2584,7 @@ bool CConnman::InitBinds(const Options& options) bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) { + AssertLockNotHeld(m_total_bytes_sent_mutex); Init(connOptions); if (fListen && !InitBinds(connOptions)) { @@ -2872,7 +2883,7 @@ void CConnman::GetNodeStats(std::vector<CNodeStats>& vstats) const for (CNode* pnode : m_nodes) { vstats.emplace_back(); pnode->CopyStats(vstats.back()); - vstats.back().m_mapped_as = pnode->addr.GetMappedAS(addrman.GetAsmap()); + vstats.back().m_mapped_as = m_netgroupman.GetMappedAS(pnode->addr); } } @@ -2926,7 +2937,9 @@ void CConnman::RecordBytesRecv(uint64_t bytes) void CConnman::RecordBytesSent(uint64_t bytes) { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); + nTotalBytesSent += bytes; const auto now = GetTime<std::chrono::seconds>(); @@ -2942,7 +2955,8 @@ void CConnman::RecordBytesSent(uint64_t bytes) uint64_t CConnman::GetMaxOutboundTarget() const { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); return nMaxOutboundLimit; } @@ -2953,7 +2967,15 @@ std::chrono::seconds CConnman::GetMaxOutboundTimeframe() const std::chrono::seconds CConnman::GetMaxOutboundTimeLeftInCycle() const { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); + return GetMaxOutboundTimeLeftInCycle_(); +} + +std::chrono::seconds CConnman::GetMaxOutboundTimeLeftInCycle_() const +{ + AssertLockHeld(m_total_bytes_sent_mutex); + if (nMaxOutboundLimit == 0) return 0s; @@ -2967,14 +2989,15 @@ std::chrono::seconds CConnman::GetMaxOutboundTimeLeftInCycle() const bool CConnman::OutboundTargetReached(bool historicalBlockServingLimit) const { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); if (nMaxOutboundLimit == 0) return false; if (historicalBlockServingLimit) { // keep a large enough buffer to at least relay each block once - const std::chrono::seconds timeLeftInCycle = GetMaxOutboundTimeLeftInCycle(); + const std::chrono::seconds timeLeftInCycle = GetMaxOutboundTimeLeftInCycle_(); const uint64_t buffer = timeLeftInCycle / std::chrono::minutes{10} * MAX_BLOCK_SERIALIZED_SIZE; if (buffer >= nMaxOutboundLimit || nMaxOutboundTotalBytesSentInCycle >= nMaxOutboundLimit - buffer) return true; @@ -2987,7 +3010,8 @@ bool CConnman::OutboundTargetReached(bool historicalBlockServingLimit) const uint64_t CConnman::GetOutboundTargetBytesLeft() const { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); if (nMaxOutboundLimit == 0) return 0; @@ -3001,7 +3025,8 @@ uint64_t CConnman::GetTotalBytesRecv() const uint64_t CConnman::GetTotalBytesSent() const { - LOCK(cs_totalBytesSent); + AssertLockNotHeld(m_total_bytes_sent_mutex); + LOCK(m_total_bytes_sent_mutex); return nTotalBytesSent; } @@ -3048,6 +3073,7 @@ bool CConnman::NodeFullyConnected(const CNode* pnode) void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { + AssertLockNotHeld(m_total_bytes_sent_mutex); size_t nMessageSize = msg.data.size(); LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", msg.m_type, nMessageSize, pnode->GetId()); if (gArgs.GetBoolArg("-capturemessages", false)) { @@ -3105,9 +3131,9 @@ CSipHasher CConnman::GetDeterministicRandomizer(uint64_t id) const return CSipHasher(nSeed0, nSeed1).Write(id); } -uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const +uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& address) const { - std::vector<unsigned char> vchNetGroup(ad.GetGroup(addrman.GetAsmap())); + std::vector<unsigned char> vchNetGroup(m_netgroupman.GetGroup(address)); return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } @@ -17,6 +17,7 @@ #include <net_permissions.h> #include <netaddress.h> #include <netbase.h> +#include <netgroup.h> #include <policy/feerate.h> #include <protocol.h> #include <random.h> @@ -760,7 +761,10 @@ public: bool m_i2p_accept_incoming; }; - void Init(const Options& connOptions) { + void Init(const Options& connOptions) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex) + { + AssertLockNotHeld(m_total_bytes_sent_mutex); + nLocalServices = connOptions.nLocalServices; nMaxConnections = connOptions.nMaxConnections; m_max_outbound_full_relay = std::min(connOptions.m_max_outbound_full_relay, connOptions.nMaxConnections); @@ -776,7 +780,7 @@ public: nReceiveFloodSize = connOptions.nReceiveFloodSize; m_peer_connect_timeout = std::chrono::seconds{connOptions.m_peer_connect_timeout}; { - LOCK(cs_totalBytesSent); + LOCK(m_total_bytes_sent_mutex); nMaxOutboundLimit = connOptions.nMaxOutboundLimit; } vWhitelistedRange = connOptions.vWhitelistedRange; @@ -787,9 +791,11 @@ public: m_onion_binds = connOptions.onion_binds; } - CConnman(uint64_t seed0, uint64_t seed1, AddrMan& addrman, bool network_active = true); + CConnman(uint64_t seed0, uint64_t seed1, AddrMan& addrman, const NetGroupManager& netgroupman, + bool network_active = true); + ~CConnman(); - bool Start(CScheduler& scheduler, const Options& options); + bool Start(CScheduler& scheduler, const Options& options) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); void StopThreads(); void StopNodes(); @@ -808,7 +814,7 @@ public: bool ForNode(NodeId id, std::function<bool(CNode* pnode)> func); - void PushMessage(CNode* pnode, CSerializedNetMsg&& msg); + void PushMessage(CNode* pnode, CSerializedNetMsg&& msg) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); using NodeFn = std::function<void(CNode*)>; void ForEachNode(const NodeFn& func) @@ -899,24 +905,22 @@ public: //! that peer during `net_processing.cpp:PushNodeVersion()`. ServiceFlags GetLocalServices() const; - uint64_t GetMaxOutboundTarget() const; + uint64_t GetMaxOutboundTarget() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); std::chrono::seconds GetMaxOutboundTimeframe() const; //! check if the outbound target is reached //! if param historicalBlockServingLimit is set true, the function will //! response true if the limit for serving historical blocks has been reached - bool OutboundTargetReached(bool historicalBlockServingLimit) const; + bool OutboundTargetReached(bool historicalBlockServingLimit) const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); //! response the bytes left in the current max outbound cycle //! in case of no limit, it will always response 0 - uint64_t GetOutboundTargetBytesLeft() const; + uint64_t GetOutboundTargetBytesLeft() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); - //! returns the time left in the current max outbound cycle - //! in case of no limit, it will always return 0 - std::chrono::seconds GetMaxOutboundTimeLeftInCycle() const; + std::chrono::seconds GetMaxOutboundTimeLeftInCycle() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); uint64_t GetTotalBytesRecv() const; - uint64_t GetTotalBytesSent() const; + uint64_t GetTotalBytesSent() const EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); /** Get a unique deterministic randomizer. */ CSipHasher GetDeterministicRandomizer(uint64_t id) const; @@ -942,6 +946,10 @@ private: NetPermissionFlags m_permissions; }; + //! returns the time left in the current max outbound cycle + //! in case of no limit, it will always return 0 + std::chrono::seconds GetMaxOutboundTimeLeftInCycle_() const EXCLUSIVE_LOCKS_REQUIRED(m_total_bytes_sent_mutex); + bool BindListenPort(const CService& bindAddr, bilingual_str& strError, NetPermissionFlags permissions); bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); bool InitBinds(const Options& options); @@ -1001,7 +1009,7 @@ private: /** * Check connected and listening sockets for IO readiness and process them accordingly. */ - void SocketHandler(); + void SocketHandler() EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); /** * Do the read/write for connected sockets that are ready for IO. @@ -1014,7 +1022,8 @@ private: void SocketHandlerConnected(const std::vector<CNode*>& nodes, const std::set<SOCKET>& recv_set, const std::set<SOCKET>& send_set, - const std::set<SOCKET>& error_set); + const std::set<SOCKET>& error_set) + EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); /** * Accept incoming connections, one from each read-ready listening socket. @@ -1022,7 +1031,7 @@ private: */ void SocketHandlerListening(const std::set<SOCKET>& recv_set); - void ThreadSocketHandler(); + void ThreadSocketHandler() EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); void ThreadDNSAddressSeed(); uint64_t CalculateKeyedNetGroup(const CAddress& ad) const; @@ -1051,7 +1060,7 @@ private: // Network stats void RecordBytesRecv(uint64_t bytes); - void RecordBytesSent(uint64_t bytes); + void RecordBytesSent(uint64_t bytes) EXCLUSIVE_LOCKS_REQUIRED(!m_total_bytes_sent_mutex); /** * Return vector of current BLOCK_RELAY peers. @@ -1062,14 +1071,14 @@ private: static bool NodeFullyConnected(const CNode* pnode); // Network usage totals - mutable RecursiveMutex cs_totalBytesSent; + mutable Mutex m_total_bytes_sent_mutex; std::atomic<uint64_t> nTotalBytesRecv{0}; - uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent) {0}; + uint64_t nTotalBytesSent GUARDED_BY(m_total_bytes_sent_mutex) {0}; // outbound limit & stats - uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(cs_totalBytesSent) {0}; - std::chrono::seconds nMaxOutboundCycleStartTime GUARDED_BY(cs_totalBytesSent) {0}; - uint64_t nMaxOutboundLimit GUARDED_BY(cs_totalBytesSent); + uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(m_total_bytes_sent_mutex) {0}; + std::chrono::seconds nMaxOutboundCycleStartTime GUARDED_BY(m_total_bytes_sent_mutex) {0}; + uint64_t nMaxOutboundLimit GUARDED_BY(m_total_bytes_sent_mutex); // P2P timeout in seconds std::chrono::seconds m_peer_connect_timeout; @@ -1085,6 +1094,7 @@ private: std::atomic<bool> fNetworkActive{true}; bool fAddressesInitialized{false}; AddrMan& addrman; + const NetGroupManager& m_netgroupman; std::deque<std::string> m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); Mutex m_addr_fetches_mutex; std::vector<std::string> m_added_nodes GUARDED_BY(m_added_nodes_mutex); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 02eb3762b1..e9e6544069 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -135,6 +135,8 @@ static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; static constexpr auto AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24h}; /** Average delay between peer address broadcasts */ static constexpr auto AVG_ADDRESS_BROADCAST_INTERVAL{30s}; +/** Delay between rotating the peers we relay a particular address to */ +static constexpr auto ROTATE_ADDR_RELAY_DEST_INTERVAL{24h}; /** Average delay between trickled inventory transmissions for inbound peers. * Blocks and peers with NetPermissionFlags::NoBan permission bypass this. */ static constexpr auto INBOUND_INVENTORY_BROADCAST_INTERVAL{5s}; @@ -1818,9 +1820,12 @@ void PeerManagerImpl::RelayAddress(NodeId originator, // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the m_addr_knowns of the chosen nodes prevent repeats const uint64_t hash_addr{CServiceHash(0, 0)(addr)}; + const auto current_time{GetTime<std::chrono::seconds>()}; + // Adding address hash makes exact rotation time different per address, while preserving periodicity. + const uint64_t time_addr{(static_cast<uint64_t>(count_seconds(current_time)) + hash_addr) / count_seconds(ROTATE_ADDR_RELAY_DEST_INTERVAL)}; const CSipHasher hasher{m_connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY) .Write(hash_addr) - .Write((GetTime() + hash_addr) / (24 * 60 * 60))}; + .Write(time_addr)}; FastRandomContext insecure_rand; // Relay reachable addresses to 2 peers. Unreachable addresses are relayed randomly to 1 or 2 peers. @@ -4472,7 +4477,7 @@ void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::mic if (pingSend) { uint64_t nonce = 0; while (nonce == 0) { - GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); + GetRandBytes({(unsigned char*)&nonce, sizeof(nonce)}); } peer.m_ping_queued = false; peer.m_ping_start = now; diff --git a/src/netaddress.cpp b/src/netaddress.cpp index f7640329f8..7bf11a3e48 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -10,7 +10,6 @@ #include <hash.h> #include <prevector.h> #include <tinyformat.h> -#include <util/asmap.h> #include <util/strencodings.h> #include <util/string.h> @@ -211,7 +210,7 @@ static void Checksum(Span<const uint8_t> addr_pubkey, uint8_t (&checksum)[CHECKS bool CNetAddr::SetSpecial(const std::string& addr) { - if (!ValidAsCString(addr)) { + if (!ContainsNoNUL(addr)) { return false; } @@ -235,17 +234,16 @@ bool CNetAddr::SetTor(const std::string& addr) return false; } - bool invalid; - const auto& input = DecodeBase32(addr.substr(0, addr.size() - suffix_len).c_str(), &invalid); + auto input = DecodeBase32(std::string_view{addr}.substr(0, addr.size() - suffix_len)); - if (invalid) { + if (!input) { return false; } - if (input.size() == torv3::TOTAL_LEN) { - Span<const uint8_t> input_pubkey{input.data(), ADDR_TORV3_SIZE}; - Span<const uint8_t> input_checksum{input.data() + ADDR_TORV3_SIZE, torv3::CHECKSUM_LEN}; - Span<const uint8_t> input_version{input.data() + ADDR_TORV3_SIZE + torv3::CHECKSUM_LEN, sizeof(torv3::VERSION)}; + if (input->size() == torv3::TOTAL_LEN) { + Span<const uint8_t> input_pubkey{input->data(), ADDR_TORV3_SIZE}; + Span<const uint8_t> input_checksum{input->data() + ADDR_TORV3_SIZE, torv3::CHECKSUM_LEN}; + Span<const uint8_t> input_version{input->data() + ADDR_TORV3_SIZE + torv3::CHECKSUM_LEN, sizeof(torv3::VERSION)}; if (input_version != torv3::VERSION) { return false; @@ -281,15 +279,14 @@ bool CNetAddr::SetI2P(const std::string& addr) // can decode it. const std::string b32_padded = addr.substr(0, b32_len) + "===="; - bool invalid; - const auto& address_bytes = DecodeBase32(b32_padded.c_str(), &invalid); + auto address_bytes = DecodeBase32(b32_padded); - if (invalid || address_bytes.size() != ADDR_I2P_SIZE) { + if (!address_bytes || address_bytes->size() != ADDR_I2P_SIZE) { return false; } m_net = NET_I2P; - m_addr.assign(address_bytes.begin(), address_bytes.end()); + m_addr.assign(address_bytes->begin(), address_bytes->end()); return true; } @@ -722,107 +719,6 @@ Network CNetAddr::GetNetClass() const return m_net; } -uint32_t CNetAddr::GetMappedAS(const std::vector<bool> &asmap) const { - uint32_t net_class = GetNetClass(); - if (asmap.size() == 0 || (net_class != NET_IPV4 && net_class != NET_IPV6)) { - return 0; // Indicates not found, safe because AS0 is reserved per RFC7607. - } - std::vector<bool> ip_bits(128); - if (HasLinkedIPv4()) { - // For lookup, treat as if it was just an IPv4 address (IPV4_IN_IPV6_PREFIX + IPv4 bits) - for (int8_t byte_i = 0; byte_i < 12; ++byte_i) { - for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { - ip_bits[byte_i * 8 + bit_i] = (IPV4_IN_IPV6_PREFIX[byte_i] >> (7 - bit_i)) & 1; - } - } - uint32_t ipv4 = GetLinkedIPv4(); - for (int i = 0; i < 32; ++i) { - ip_bits[96 + i] = (ipv4 >> (31 - i)) & 1; - } - } else { - // Use all 128 bits of the IPv6 address otherwise - assert(IsIPv6()); - for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { - uint8_t cur_byte = m_addr[byte_i]; - for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { - ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; - } - } - } - uint32_t mapped_as = Interpret(asmap, ip_bits); - return mapped_as; -} - -/** - * Get the canonical identifier of our network group - * - * The groups are assigned in a way where it should be costly for an attacker to - * obtain addresses with many different group identifiers, even if it is cheap - * to obtain addresses with the same identifier. - * - * @note No two connections will be attempted to addresses with the same network - * group. - */ -std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) const -{ - std::vector<unsigned char> vchRet; - uint32_t net_class = GetNetClass(); - // If non-empty asmap is supplied and the address is IPv4/IPv6, - // return ASN to be used for bucketing. - uint32_t asn = GetMappedAS(asmap); - if (asn != 0) { // Either asmap was empty, or address has non-asmappable net class (e.g. TOR). - vchRet.push_back(NET_IPV6); // IPv4 and IPv6 with same ASN should be in the same bucket - for (int i = 0; i < 4; i++) { - vchRet.push_back((asn >> (8 * i)) & 0xFF); - } - return vchRet; - } - - vchRet.push_back(net_class); - int nBits{0}; - - if (IsLocal()) { - // all local addresses belong to the same group - } else if (IsInternal()) { - // all internal-usage addresses get their own group - nBits = ADDR_INTERNAL_SIZE * 8; - } else if (!IsRoutable()) { - // all other unroutable addresses belong to the same group - } else if (HasLinkedIPv4()) { - // IPv4 addresses (and mapped IPv4 addresses) use /16 groups - uint32_t ipv4 = GetLinkedIPv4(); - vchRet.push_back((ipv4 >> 24) & 0xFF); - vchRet.push_back((ipv4 >> 16) & 0xFF); - return vchRet; - } else if (IsTor() || IsI2P()) { - nBits = 4; - } else if (IsCJDNS()) { - // Treat in the same way as Tor and I2P because the address in all of - // them is "random" bytes (derived from a public key). However in CJDNS - // the first byte is a constant 0xfc, so the random bytes come after it. - // Thus skip the constant 8 bits at the start. - nBits = 12; - } else if (IsHeNet()) { - // for he.net, use /36 groups - nBits = 36; - } else { - // for the rest of the IPv6 network, use /32 groups - nBits = 32; - } - - // Push our address onto vchRet. - const size_t num_bytes = nBits / 8; - vchRet.insert(vchRet.end(), m_addr.begin(), m_addr.begin() + num_bytes); - nBits %= 8; - // ...for the last byte, push nBits and for the rest of the byte push 1's - if (nBits > 0) { - assert(num_bytes < m_addr.size()); - vchRet.push_back(m_addr[num_bytes] | ((1 << (8 - nBits)) - 1)); - } - - return vchRet; -} - std::vector<unsigned char> CNetAddr::GetAddrBytes() const { if (IsAddrV1Compatible()) { diff --git a/src/netaddress.h b/src/netaddress.h index 6d21dcd5cd..b9a8dc589a 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -202,12 +202,6 @@ public: //! Whether this address has a linked IPv4 address (see GetLinkedIPv4()). bool HasLinkedIPv4() const; - // The AS on the BGP path to the node we use to diversify - // peers in AddrMan bucketing based on the AS infrastructure. - // The ip->AS mapping depends on how asmap is constructed. - uint32_t GetMappedAS(const std::vector<bool>& asmap) const; - - std::vector<unsigned char> GetGroup(const std::vector<bool>& asmap) const; std::vector<unsigned char> GetAddrBytes() const; int GetReachabilityFrom(const CNetAddr* paddrPartner = nullptr) const; diff --git a/src/netbase.cpp b/src/netbase.cpp index 9557297df1..8ff3b7a68c 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -136,7 +136,7 @@ static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, un { vIP.clear(); - if (!ValidAsCString(name)) { + if (!ContainsNoNUL(name)) { return false; } @@ -169,7 +169,7 @@ static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, un bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ValidAsCString(name)) { + if (!ContainsNoNUL(name)) { return false; } std::string strHost = name; @@ -184,7 +184,7 @@ bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned in bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ValidAsCString(name)) { + if (!ContainsNoNUL(name)) { return false; } std::vector<CNetAddr> vIP; @@ -197,7 +197,7 @@ bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSL bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function) { - if (name.empty() || !ValidAsCString(name)) { + if (name.empty() || !ContainsNoNUL(name)) { return false; } uint16_t port{portDefault}; @@ -216,7 +216,7 @@ bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t port bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ValidAsCString(name)) { + if (!ContainsNoNUL(name)) { return false; } std::vector<CService> vService; @@ -229,7 +229,7 @@ bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupFn dns_lookup_function) { - if (!ValidAsCString(name)) { + if (!ContainsNoNUL(name)) { return {}; } CService addr; @@ -675,7 +675,7 @@ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_ return false; } } else { - if (!Socks5(strDest, port, 0, sock)) { + if (!Socks5(strDest, port, nullptr, sock)) { return false; } } @@ -684,7 +684,7 @@ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) { - if (!ValidAsCString(subnet_str)) { + if (!ContainsNoNUL(subnet_str)) { return false; } diff --git a/src/netgroup.cpp b/src/netgroup.cpp new file mode 100644 index 0000000000..5f42d6c719 --- /dev/null +++ b/src/netgroup.cpp @@ -0,0 +1,111 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <netgroup.h> + +#include <hash.h> +#include <util/asmap.h> + +uint256 NetGroupManager::GetAsmapChecksum() const +{ + if (!m_asmap.size()) return {}; + + return SerializeHash(m_asmap); +} + +std::vector<unsigned char> NetGroupManager::GetGroup(const CNetAddr& address) const +{ + std::vector<unsigned char> vchRet; + // If non-empty asmap is supplied and the address is IPv4/IPv6, + // return ASN to be used for bucketing. + uint32_t asn = GetMappedAS(address); + if (asn != 0) { // Either asmap was empty, or address has non-asmappable net class (e.g. TOR). + vchRet.push_back(NET_IPV6); // IPv4 and IPv6 with same ASN should be in the same bucket + for (int i = 0; i < 4; i++) { + vchRet.push_back((asn >> (8 * i)) & 0xFF); + } + return vchRet; + } + + vchRet.push_back(address.GetNetClass()); + int nStartByte{0}; + int nBits{0}; + + if (address.IsLocal()) { + // all local addresses belong to the same group + } else if (address.IsInternal()) { + // All internal-usage addresses get their own group. + // Skip over the INTERNAL_IN_IPV6_PREFIX returned by CAddress::GetAddrBytes(). + nStartByte = INTERNAL_IN_IPV6_PREFIX.size(); + nBits = ADDR_INTERNAL_SIZE * 8; + } else if (!address.IsRoutable()) { + // all other unroutable addresses belong to the same group + } else if (address.HasLinkedIPv4()) { + // IPv4 addresses (and mapped IPv4 addresses) use /16 groups + uint32_t ipv4 = address.GetLinkedIPv4(); + vchRet.push_back((ipv4 >> 24) & 0xFF); + vchRet.push_back((ipv4 >> 16) & 0xFF); + return vchRet; + } else if (address.IsTor() || address.IsI2P()) { + nBits = 4; + } else if (address.IsCJDNS()) { + // Treat in the same way as Tor and I2P because the address in all of + // them is "random" bytes (derived from a public key). However in CJDNS + // the first byte is a constant 0xfc, so the random bytes come after it. + // Thus skip the constant 8 bits at the start. + nBits = 12; + } else if (address.IsHeNet()) { + // for he.net, use /36 groups + nBits = 36; + } else { + // for the rest of the IPv6 network, use /32 groups + nBits = 32; + } + + // Push our address onto vchRet. + auto addr_bytes = address.GetAddrBytes(); + const size_t num_bytes = nBits / 8; + vchRet.insert(vchRet.end(), addr_bytes.begin() + nStartByte, addr_bytes.begin() + nStartByte + num_bytes); + nBits %= 8; + // ...for the last byte, push nBits and for the rest of the byte push 1's + if (nBits > 0) { + assert(num_bytes < addr_bytes.size()); + vchRet.push_back(addr_bytes[num_bytes] | ((1 << (8 - nBits)) - 1)); + } + + return vchRet; +} + +uint32_t NetGroupManager::GetMappedAS(const CNetAddr& address) const +{ + uint32_t net_class = address.GetNetClass(); + if (m_asmap.size() == 0 || (net_class != NET_IPV4 && net_class != NET_IPV6)) { + return 0; // Indicates not found, safe because AS0 is reserved per RFC7607. + } + std::vector<bool> ip_bits(128); + if (address.HasLinkedIPv4()) { + // For lookup, treat as if it was just an IPv4 address (IPV4_IN_IPV6_PREFIX + IPv4 bits) + for (int8_t byte_i = 0; byte_i < 12; ++byte_i) { + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (IPV4_IN_IPV6_PREFIX[byte_i] >> (7 - bit_i)) & 1; + } + } + uint32_t ipv4 = address.GetLinkedIPv4(); + for (int i = 0; i < 32; ++i) { + ip_bits[96 + i] = (ipv4 >> (31 - i)) & 1; + } + } else { + // Use all 128 bits of the IPv6 address otherwise + assert(address.IsIPv6()); + auto addr_bytes = address.GetAddrBytes(); + for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { + uint8_t cur_byte = addr_bytes[byte_i]; + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; + } + } + } + uint32_t mapped_as = Interpret(m_asmap, ip_bits); + return mapped_as; +} diff --git a/src/netgroup.h b/src/netgroup.h new file mode 100644 index 0000000000..2dd63ec66b --- /dev/null +++ b/src/netgroup.h @@ -0,0 +1,66 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NETGROUP_H +#define BITCOIN_NETGROUP_H + +#include <netaddress.h> +#include <uint256.h> + +#include <vector> + +/** + * Netgroup manager + */ +class NetGroupManager { +public: + explicit NetGroupManager(std::vector<bool> asmap) + : m_asmap{std::move(asmap)} + {} + + /** Get a checksum identifying the asmap being used. */ + uint256 GetAsmapChecksum() const; + + /** + * Get the canonical identifier of the network group for address. + * + * The groups are assigned in a way where it should be costly for an attacker to + * obtain addresses with many different group identifiers, even if it is cheap + * to obtain addresses with the same identifier. + * + * @note No two connections will be attempted to addresses with the same network + * group. + */ + std::vector<unsigned char> GetGroup(const CNetAddr& address) const; + + /** + * Get the autonomous system on the BGP path to address. + * + * The ip->AS mapping depends on how asmap is constructed. + */ + uint32_t GetMappedAS(const CNetAddr& address) const; + +private: + /** Compressed IP->ASN mapping, loaded from a file when a node starts. + * + * This mapping is then used for bucketing nodes in Addrman and for + * ensuring we connect to a diverse set of peers in Connman. The map is + * empty if no file was provided. + * + * If asmap is provided, nodes will be bucketed by AS they belong to, in + * order to make impossible for a node to connect to several nodes hosted + * in a single AS. This is done in response to Erebus attack, but also to + * generally diversify the connections every node creates, especially + * useful when a large fraction of nodes operate under a couple of cloud + * providers. + * + * If a new asmap is provided, the existing addrman records are + * re-bucketed. + * + * This is initialized in the constructor, const, and therefore is + * thread-safe. */ + const std::vector<bool> m_asmap; +}; + +#endif // BITCOIN_NETGROUP_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 21cb0250d8..25771aacee 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -21,6 +21,8 @@ #include <util/system.h> #include <validation.h> +#include <unordered_map> + namespace node { std::atomic_bool fImporting(false); std::atomic_bool fReindex(false); @@ -230,6 +232,11 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr nLastBlockWeCanPrune, count); } +void BlockManager::UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) { + AssertLockHeld(::cs_main); + m_prune_locks[name] = lock_info; +} + CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); @@ -290,20 +297,6 @@ bool BlockManager::LoadBlockIndex(const Consensus::Params& consensus_params) return true; } -void BlockManager::Unload() -{ - m_blocks_unlinked.clear(); - - m_block_index.clear(); - - m_blockfile_info.clear(); - m_last_blockfile = 0; - m_dirty_blockindex.clear(); - m_dirty_fileinfo.clear(); - - m_have_pruned = false; -} - bool BlockManager::WriteBlockIndexDB() { AssertLockHeld(::cs_main); @@ -397,6 +390,16 @@ bool BlockManager::IsBlockPruned(const CBlockIndex* pblockindex) return (m_have_pruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); } +const CBlockIndex* GetFirstStoredBlock(const CBlockIndex* start_block) { + AssertLockHeld(::cs_main); + assert(start_block); + const CBlockIndex* last_block = start_block; + while (last_block->pprev && (last_block->pprev->nStatus & BLOCK_HAVE_DATA)) { + last_block = last_block->pprev; + } + return last_block; +} + // If we're using -prune with -reindex, then delete block files that will be ignored by the // reindex. Since reindexing works by starting at block file 0 and looping until a blockfile // is missing, do the same here to delete any later block files after a gap. Also delete all diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 11445aa22e..622eac7fef 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -13,6 +13,7 @@ #include <atomic> #include <cstdint> +#include <unordered_map> #include <vector> extern RecursiveMutex cs_main; @@ -65,6 +66,10 @@ struct CBlockIndexHeightOnlyComparator { bool operator()(const CBlockIndex* pa, const CBlockIndex* pb) const; }; +struct PruneLockInfo { + int height_first{std::numeric_limits<int>::max()}; //! Height of earliest block that should be kept and not pruned +}; + /** * Maintains a tree of blocks (stored in `m_block_index`) which is consulted * to determine where the most-work tip is. @@ -78,6 +83,13 @@ class BlockManager friend ChainstateManager; private: + /** + * Load the blocktree off disk and into memory. Populate certain metadata + * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral + * collections like m_dirty_blockindex. + */ + bool LoadBlockIndex(const Consensus::Params& consensus_params) + EXCLUSIVE_LOCKS_REQUIRED(cs_main); void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); void FlushUndoFile(int block_file, bool finalize = false); bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown); @@ -118,6 +130,14 @@ private: /** Dirty block file entries. */ std::set<int> m_dirty_fileinfo; + /** + * Map from external index name to oldest block that must not be pruned. + * + * @note Internally, only blocks at height (height_first - PRUNE_LOCK_BUFFER - 1) and + * below will be pruned, but callers should avoid assuming any particular buffer size. + */ + std::unordered_map<std::string, PruneLockInfo> m_prune_locks GUARDED_BY(::cs_main); + public: BlockMap m_block_index GUARDED_BY(cs_main); @@ -134,17 +154,6 @@ public: bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - /** - * Load the blocktree off disk and into memory. Populate certain metadata - * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral - * collections like m_dirty_blockindex. - */ - bool LoadBlockIndex(const Consensus::Params& consensus_params) - EXCLUSIVE_LOCKS_REQUIRED(cs_main); - - /** Clear all data members. */ - void Unload() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - CBlockIndex* AddToBlockIndex(const CBlockHeader& block, CBlockIndex*& best_header) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Create a new block index entry for a given block hash */ CBlockIndex* InsertBlockIndex(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -175,12 +184,13 @@ public: //! Check whether the block associated with this index entry is pruned or not. bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - ~BlockManager() - { - Unload(); - } + //! Create or update a prune lock identified by its name + void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); }; +//! Find the first block that is not pruned +const CBlockIndex* GetFirstStoredBlock(const CBlockIndex* start_block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void CleanupBlockRevFiles(); /** Open a block file (blk?????.dat) */ diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index e43211402c..99615dea69 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -32,8 +32,6 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, chainman.m_total_coinstip_cache = nCoinCacheUsage; chainman.m_total_coinsdb_cache = nCoinDBCache; - UnloadBlockIndex(mempool, chainman); - auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: diff --git a/src/node/context.cpp b/src/node/context.cpp index 893c32f1bc..0b31c10f44 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -9,6 +9,7 @@ #include <interfaces/chain.h> #include <net.h> #include <net_processing.h> +#include <netgroup.h> #include <policy/fees.h> #include <scheduler.h> #include <txmempool.h> diff --git a/src/node/context.h b/src/node/context.h index 644c997531..91ba456219 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -18,6 +18,7 @@ class CConnman; class CScheduler; class CTxMemPool; class ChainstateManager; +class NetGroupManager; class PeerManager; namespace interfaces { class Chain; @@ -43,6 +44,7 @@ struct NodeContext { std::unique_ptr<AddrMan> addrman; std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; + std::unique_ptr<const NetGroupManager> netgroupman; std::unique_ptr<CBlockPolicyEstimator> fee_estimator; std::unique_ptr<PeerManager> peerman; std::unique_ptr<ChainstateManager> chainman; diff --git a/src/psbt.cpp b/src/psbt.cpp index c8c73e130b..6465e353be 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -388,18 +388,17 @@ std::string PSBTRoleName(PSBTRole role) { bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error) { - bool invalid; - std::string tx_data = DecodeBase64(base64_tx, &invalid); - if (invalid) { + auto tx_data = DecodeBase64(base64_tx); + if (!tx_data) { error = "invalid base64"; return false; } - return DecodeRawPSBT(psbt, tx_data, error); + return DecodeRawPSBT(psbt, MakeByteSpan(*tx_data), error); } -bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) +bool DecodeRawPSBT(PartiallySignedTransaction& psbt, Span<const std::byte> tx_data, std::string& error) { - CDataStream ss_data(MakeByteSpan(tx_data), SER_NETWORK, PROTOCOL_VERSION); + CDataStream ss_data(tx_data, SER_NETWORK, PROTOCOL_VERSION); try { ss_data >> psbt; if (!ss_data.empty()) { diff --git a/src/psbt.h b/src/psbt.h index f0ceb02481..8a9cbd33d2 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -988,6 +988,6 @@ bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransacti //! Decode a base64ed PSBT into a PartiallySignedTransaction [[nodiscard]] bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); //! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction -[[nodiscard]] bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); +[[nodiscard]] bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, Span<const std::byte> raw_psbt, std::string& error); #endif // BITCOIN_PSBT_H diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 76e5207d81..e38ac6026a 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -361,18 +361,6 @@ namespace GUIUtil #endif } - /** - * Queue a function to run in an object's event loop. This can be - * replaced by a call to the QMetaObject::invokeMethod functor overload after Qt 5.10, but - * for now use a QObject::connect for compatibility with older Qt versions, based on - * https://stackoverflow.com/questions/21646467/how-to-execute-a-functor-or-a-lambda-in-a-given-thread-in-qt-gcd-style - */ - template <typename Fn> - void ObjectInvoke(QObject* object, Fn&& function, Qt::ConnectionType connection = Qt::QueuedConnection) - { - QObject source; - QObject::connect(&source, &QObject::destroyed, object, std::forward<Fn>(function), connection); - } /** * Replaces a plain text link with an HTML tagged one. diff --git a/src/qt/initexecutor.cpp b/src/qt/initexecutor.cpp index 24ae7ba73d..d269dfec71 100644 --- a/src/qt/initexecutor.cpp +++ b/src/qt/initexecutor.cpp @@ -5,13 +5,13 @@ #include <qt/initexecutor.h> #include <interfaces/node.h> -#include <qt/guiutil.h> #include <util/system.h> #include <util/threadnames.h> #include <exception> #include <QDebug> +#include <QMetaObject> #include <QObject> #include <QString> #include <QThread> @@ -39,7 +39,7 @@ void InitExecutor::handleRunawayException(const std::exception* e) void InitExecutor::initialize() { - GUIUtil::ObjectInvoke(&m_context, [this] { + QMetaObject::invokeMethod(&m_context, [this] { try { util::ThreadRename("qt-init"); qDebug() << "Running initialization in thread"; @@ -56,7 +56,7 @@ void InitExecutor::initialize() void InitExecutor::shutdown() { - GUIUtil::ObjectInvoke(&m_context, [this] { + QMetaObject::invokeMethod(&m_context, [this] { try { qDebug() << "Running Shutdown in thread"; m_node.appShutdown(); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 63b4055092..f4928951fe 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -226,7 +226,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) } /* If current default data directory does not exist, let the user choose one */ - Intro intro(0, Params().AssumedBlockchainSize(), Params().AssumedChainStateSize()); + Intro intro(nullptr, Params().AssumedBlockchainSize(), Params().AssumedChainStateSize()); intro.setDataDirectory(dataDir); intro.setWindowIcon(QIcon(":icons/bitcoin")); did_show_intro = true; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index eb69fabe89..4a51990f88 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1032,8 +1032,9 @@ void RPCConsole::on_lineEdit_returnPressed() ui->lineEdit->clear(); + WalletModel* wallet_model{nullptr}; #ifdef ENABLE_WALLET - WalletModel* wallet_model = ui->WalletSelector->currentData().value<WalletModel*>(); + wallet_model = ui->WalletSelector->currentData().value<WalletModel*>(); if (m_last_wallet_model != wallet_model) { if (wallet_model) { @@ -1049,7 +1050,10 @@ void RPCConsole::on_lineEdit_returnPressed() //: A console message indicating an entered command is currently being executed. message(CMD_REPLY, tr("Executing…")); m_is_executing = true; - Q_EMIT cmdRequest(cmd, m_last_wallet_model); + + QMetaObject::invokeMethod(m_executor, [this, cmd, wallet_model] { + m_executor->request(cmd, wallet_model); + }); cmd = QString::fromStdString(strFilteredCmd); @@ -1091,11 +1095,11 @@ void RPCConsole::browseHistory(int offset) void RPCConsole::startExecutor() { - RPCExecutor *executor = new RPCExecutor(m_node); - executor->moveToThread(&thread); + m_executor = new RPCExecutor(m_node); + m_executor->moveToThread(&thread); // Replies from executor object must go to this object - connect(executor, &RPCExecutor::reply, this, [this](int category, const QString& command) { + connect(m_executor, &RPCExecutor::reply, this, [this](int category, const QString& command) { // Remove "Executing…" message. ui->messagesWidget->undo(); message(category, command); @@ -1103,16 +1107,13 @@ void RPCConsole::startExecutor() m_is_executing = false; }); - // Requests from this object must go to executor - connect(this, &RPCConsole::cmdRequest, executor, &RPCExecutor::request); - // Make sure executor object is deleted in its own thread - connect(&thread, &QThread::finished, executor, &RPCExecutor::deleteLater); + connect(&thread, &QThread::finished, m_executor, &RPCExecutor::deleteLater); // Default implementation of QThread::run() simply spins up an event loop in the thread, // which is what we want. thread.start(); - QTimer::singleShot(0, executor, []() { + QTimer::singleShot(0, m_executor, []() { util::ThreadRename("qt-rpcconsole"); }); } diff --git a/src/qt/rpcconsole.h b/src/qt/rpcconsole.h index 528e2bef7d..1a54fe0cad 100644 --- a/src/qt/rpcconsole.h +++ b/src/qt/rpcconsole.h @@ -5,6 +5,10 @@ #ifndef BITCOIN_QT_RPCCONSOLE_H #define BITCOIN_QT_RPCCONSOLE_H +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + #include <qt/guiutil.h> #include <qt/peertablemodel.h> @@ -17,6 +21,7 @@ class ClientModel; class PlatformStyle; +class RPCExecutor; class RPCTimerInterface; class WalletModel; @@ -49,8 +54,11 @@ public: } void setClientModel(ClientModel *model = nullptr, int bestblock_height = 0, int64_t bestblock_date = 0, double verification_progress = 0.0); - void addWallet(WalletModel * const walletModel); + +#ifdef ENABLE_WALLET + void addWallet(WalletModel* const walletModel); void removeWallet(WalletModel* const walletModel); +#endif // ENABLE_WALLET enum MessageClass { MC_ERROR, @@ -129,10 +137,6 @@ public Q_SLOTS: /** set which tab has the focus (is visible) */ void setTabFocus(enum TabTypes tabType); -Q_SIGNALS: - // For RPC command executor - void cmdRequest(const QString &command, const WalletModel* wallet_model); - private: struct TranslatedStrings { const QString yes{tr("Yes")}, no{tr("No")}, to{tr("To")}, from{tr("From")}, @@ -166,6 +170,7 @@ private: int consoleFontSize = 0; QCompleter *autoCompleter = nullptr; QThread thread; + RPCExecutor* m_executor{nullptr}; WalletModel* m_last_wallet_model{nullptr}; bool m_is_executing{false}; QByteArray m_peer_widget_header_state; diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index b025bb367c..d27ddf1aba 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -24,6 +24,7 @@ #include <QApplication> #include <QMessageBox> +#include <QMetaObject> #include <QMutexLocker> #include <QThread> #include <QTimer> @@ -135,7 +136,7 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal // handled on the GUI event loop. wallet_model->moveToThread(thread()); // setParent(parent) must be called in the thread which created the parent object. More details in #18948. - GUIUtil::ObjectInvoke(this, [wallet_model, this] { + QMetaObject::invokeMethod(this, [wallet_model, this] { wallet_model->setParent(this); }, GUIUtil::blockingGUIThreadConnection()); diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 91ce420a33..dc4e25a02b 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -194,16 +194,16 @@ void WalletFrame::gotoVerifyMessageTab(QString addr) void WalletFrame::gotoLoadPSBT(bool from_clipboard) { - std::string data; + std::vector<unsigned char> data; if (from_clipboard) { std::string raw = QApplication::clipboard()->text().toStdString(); - bool invalid; - data = DecodeBase64(raw, &invalid); - if (invalid) { + auto result = DecodeBase64(raw); + if (!result) { Q_EMIT message(tr("Error"), tr("Unable to decode PSBT from clipboard (invalid base64)"), CClientUIInterface::MSG_ERROR); return; } + data = std::move(*result); } else { QString filename = GUIUtil::getOpenFileName(this, tr("Load Transaction Data"), QString(), @@ -214,12 +214,12 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard) return; } std::ifstream in{filename.toLocal8Bit().data(), std::ios::binary}; - data = std::string(std::istreambuf_iterator<char>{in}, {}); + data.assign(std::istream_iterator<unsigned char>{in}, {}); } std::string error; PartiallySignedTransaction psbtx; - if (!DecodeRawPSBT(psbtx, data, error)) { + if (!DecodeRawPSBT(psbtx, MakeByteSpan(data), error)) { Q_EMIT message(tr("Error"), tr("Unable to decode PSBT") + "\n" + QString::fromStdString(error), CClientUIInterface::MSG_ERROR); return; } diff --git a/src/random.cpp b/src/random.cpp index 77d2ae4d43..6ae08103b1 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -16,6 +16,7 @@ #include <logging.h> #include <randomenv.h> #include <support/allocators/secure.h> +#include <span.h> #include <sync.h> // for Mutex #include <util/time.h> // for GetTimeMicros() @@ -578,8 +579,8 @@ static void ProcRand(unsigned char* out, int num, RNGLevel level) noexcept } } -void GetRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::FAST); } -void GetStrongRandBytes(unsigned char* buf, int num) noexcept { ProcRand(buf, num, RNGLevel::SLOW); } +void GetRandBytes(Span<unsigned char> bytes) noexcept { ProcRand(bytes.data(), bytes.size(), RNGLevel::FAST); } +void GetStrongRandBytes(Span<unsigned char> bytes) noexcept { ProcRand(bytes.data(), bytes.size(), RNGLevel::SLOW); } void RandAddPeriodic() noexcept { ProcRand(nullptr, 0, RNGLevel::PERIODIC); } void RandAddEvent(const uint32_t event_info) noexcept { GetRNGState().AddEvent(event_info); } @@ -598,7 +599,7 @@ int GetRandInt(int nMax) noexcept uint256 GetRandHash() noexcept { uint256 hash; - GetRandBytes((unsigned char*)&hash, sizeof(hash)); + GetRandBytes(hash); return hash; } diff --git a/src/random.h b/src/random.h index 97302d61ab..285158b1c3 100644 --- a/src/random.h +++ b/src/random.h @@ -8,6 +8,7 @@ #include <crypto/chacha20.h> #include <crypto/common.h> +#include <span.h> #include <uint256.h> #include <chrono> @@ -66,7 +67,7 @@ * * Thread-safe. */ -void GetRandBytes(unsigned char* buf, int num) noexcept; +void GetRandBytes(Span<unsigned char> bytes) noexcept; /** Generate a uniform random integer in the range [0..range). Precondition: range > 0 */ uint64_t GetRand(uint64_t nMax) noexcept; /** Generate a uniform random duration in the range [0..max). Precondition: max.count() > 0 */ @@ -105,7 +106,7 @@ uint256 GetRandHash() noexcept; * * Thread-safe. */ -void GetStrongRandBytes(unsigned char* buf, int num) noexcept; +void GetStrongRandBytes(Span<unsigned char> bytes) noexcept; /** * Gather entropy from various expensive sources, and feed them to the PRNG state. diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 388841289a..53ebeb31cd 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -363,10 +363,10 @@ void RandAddStaticEnv(CSHA512& hasher) #if HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS // Network interfaces - struct ifaddrs *ifad = NULL; + struct ifaddrs *ifad = nullptr; getifaddrs(&ifad); struct ifaddrs *ifit = ifad; - while (ifit != NULL) { + while (ifit != nullptr) { hasher.Write((const unsigned char*)&ifit, sizeof(ifit)); hasher.Write((const unsigned char*)ifit->ifa_name, strlen(ifit->ifa_name) + 1); hasher.Write((const unsigned char*)&ifit->ifa_flags, sizeof(ifit->ifa_flags)); diff --git a/src/rest.cpp b/src/rest.cpp index 2aeb9c68c3..22b5d2e074 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -32,8 +32,6 @@ #include <any> #include <string> -#include <boost/algorithm/string.hpp> - #include <univalue.h> using node::GetTransaction; @@ -191,8 +189,7 @@ static bool rest_headers(const std::any& context, return false; std::string param; const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); - std::vector<std::string> path; - boost::split(path, param, boost::is_any_of("/")); + std::vector<std::string> path = SplitString(param, '/'); std::string raw_count; std::string hashStr; @@ -362,8 +359,7 @@ static bool rest_filter_header(const std::any& context, HTTPRequest* req, const std::string param; const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); - std::vector<std::string> uri_parts; - boost::split(uri_parts, param, boost::is_any_of("/")); + std::vector<std::string> uri_parts = SplitString(param, '/'); std::string raw_count; std::string raw_blockhash; if (uri_parts.size() == 3) { @@ -483,8 +479,7 @@ static bool rest_block_filter(const std::any& context, HTTPRequest* req, const s const RESTResponseFormat rf = ParseDataFormat(param, strURIPart); // request is sent over URI scheme /rest/blockfilter/filtertype/blockhash - std::vector<std::string> uri_parts; - boost::split(uri_parts, param, boost::is_any_of("/")); + std::vector<std::string> uri_parts = SplitString(param, '/'); if (uri_parts.size() != 2) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilter/<filtertype>/<blockhash>"); } @@ -712,7 +707,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: if (param.length() > 1) { std::string strUriParams = param.substr(1); - boost::split(uriParts, strUriParams, boost::is_any_of("/")); + uriParts = SplitString(strUriParams, '/'); } // throw exception in case of an empty request diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f46e5e9fef..398b948388 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -37,6 +37,7 @@ #include <txmempool.h> #include <undo.h> #include <univalue.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/translation.h> #include <validation.h> @@ -785,12 +786,10 @@ static RPCHelpMan pruneblockchain() } PruneBlockFilesManual(active_chainstate, height); - const CBlockIndex* block = active_chain.Tip(); - CHECK_NONFATAL(block); - while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { - block = block->pprev; - } - return uint64_t(block->nHeight); + const CBlockIndex* block = CHECK_NONFATAL(active_chain.Tip()); + const CBlockIndex* last_block = node::GetFirstStoredBlock(block); + + return static_cast<uint64_t>(last_block->nHeight); }, }; } @@ -1200,8 +1199,7 @@ RPCHelpMan getblockchaininfo() LOCK(cs_main); CChainState& active_chainstate = chainman.ActiveChainstate(); - const CBlockIndex* tip = active_chainstate.m_chain.Tip(); - CHECK_NONFATAL(tip); + const CBlockIndex* tip = CHECK_NONFATAL(active_chainstate.m_chain.Tip()); const int height = tip->nHeight; UniValue obj(UniValue::VOBJ); obj.pushKV("chain", Params().NetworkIDString()); @@ -1217,13 +1215,8 @@ RPCHelpMan getblockchaininfo() obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage()); obj.pushKV("pruned", node::fPruneMode); if (node::fPruneMode) { - const CBlockIndex* block = tip; - CHECK_NONFATAL(block); - while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) { - block = block->pprev; - } - - obj.pushKV("pruneheight", block->nHeight); + const CBlockIndex* block = CHECK_NONFATAL(tip); + obj.pushKV("pruneheight", node::GetFirstStoredBlock(block)->nHeight); // if 0, execution bypasses the whole if block. bool automatic_pruning{args.GetIntArg("-prune", 0) != 1}; @@ -1309,8 +1302,7 @@ static RPCHelpMan getdeploymentinfo() const CBlockIndex* blockindex; if (request.params[0].isNull()) { - blockindex = active_chainstate.m_chain.Tip(); - CHECK_NONFATAL(blockindex); + blockindex = CHECK_NONFATAL(active_chainstate.m_chain.Tip()); } else { const uint256 hash(ParseHashV(request.params[0], "blockhash")); blockindex = chainman.m_blockman.LookupBlockIndex(hash); @@ -2131,10 +2123,8 @@ static RPCHelpMan scantxoutset() LOCK(cs_main); CChainState& active_chainstate = chainman.ActiveChainstate(); active_chainstate.ForceFlushStateToDisk(); - pcursor = active_chainstate.CoinsDB().Cursor(); - CHECK_NONFATAL(pcursor); - tip = active_chainstate.m_chain.Tip(); - CHECK_NONFATAL(tip); + pcursor = CHECK_NONFATAL(active_chainstate.CoinsDB().Cursor()); + tip = CHECK_NONFATAL(active_chainstate.m_chain.Tip()); } bool res = FindScriptPubKey(g_scan_progress, g_should_abort_scan, count, pcursor.get(), needles, coins, node.rpc_interruption_point); result.pushKV("success", res); @@ -2336,8 +2326,7 @@ UniValue CreateUTXOSnapshot( } pcursor = chainstate.CoinsDB().Cursor(); - tip = chainstate.m_blockman.LookupBlockIndex(stats.hashBlock); - CHECK_NONFATAL(tip); + tip = CHECK_NONFATAL(chainstate.m_blockman.LookupBlockIndex(stats.hashBlock)); } LOG_TIME_SECONDS(strprintf("writing UTXO snapshot at height %s (%s) to file %s (via %s)", @@ -2377,44 +2366,36 @@ UniValue CreateUTXOSnapshot( return result; } - -void RegisterBlockchainRPCCommands(CRPCTable &t) -{ -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // --------------------- ------------------------ - { "blockchain", &getblockchaininfo, }, - { "blockchain", &getchaintxstats, }, - { "blockchain", &getblockstats, }, - { "blockchain", &getbestblockhash, }, - { "blockchain", &getblockcount, }, - { "blockchain", &getblock, }, - { "blockchain", &getblockfrompeer, }, - { "blockchain", &getblockhash, }, - { "blockchain", &getblockheader, }, - { "blockchain", &getchaintips, }, - { "blockchain", &getdifficulty, }, - { "blockchain", &getdeploymentinfo, }, - { "blockchain", &gettxout, }, - { "blockchain", &gettxoutsetinfo, }, - { "blockchain", &pruneblockchain, }, - { "blockchain", &verifychain, }, - - { "blockchain", &preciousblock, }, - { "blockchain", &scantxoutset, }, - { "blockchain", &getblockfilter, }, - - /* Not shown in help */ - { "hidden", &invalidateblock, }, - { "hidden", &reconsiderblock, }, - { "hidden", &waitfornewblock, }, - { "hidden", &waitforblock, }, - { "hidden", &waitforblockheight, }, - { "hidden", &syncwithvalidationinterfacequeue, }, - { "hidden", &dumptxoutset, }, -}; -// clang-format on +void RegisterBlockchainRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + {"blockchain", &getblockchaininfo}, + {"blockchain", &getchaintxstats}, + {"blockchain", &getblockstats}, + {"blockchain", &getbestblockhash}, + {"blockchain", &getblockcount}, + {"blockchain", &getblock}, + {"blockchain", &getblockfrompeer}, + {"blockchain", &getblockhash}, + {"blockchain", &getblockheader}, + {"blockchain", &getchaintips}, + {"blockchain", &getdifficulty}, + {"blockchain", &getdeploymentinfo}, + {"blockchain", &gettxout}, + {"blockchain", &gettxoutsetinfo}, + {"blockchain", &pruneblockchain}, + {"blockchain", &verifychain}, + {"blockchain", &preciousblock}, + {"blockchain", &scantxoutset}, + {"blockchain", &getblockfilter}, + {"hidden", &invalidateblock}, + {"hidden", &reconsiderblock}, + {"hidden", &waitfornewblock}, + {"hidden", &waitforblock}, + {"hidden", &waitforblockheight}, + {"hidden", &syncwithvalidationinterfacequeue}, + {"hidden", &dumptxoutset}, + }; for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index 82aa6f9516..4de7fc4205 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -62,15 +62,11 @@ static RPCHelpMan enumeratesigners() }; } -void RegisterSignerRPCCommands(CRPCTable &t) +void RegisterSignerRPCCommands(CRPCTable& t) { -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // --------------------- ------------------------ - { "signer", &enumeratesigners, }, -}; -// clang-format on + static const CRPCCommand commands[]{ + {"signer", &enumeratesigners}, + }; for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 1caf4ad96c..27080d3881 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -672,8 +672,6 @@ static RPCHelpMan savemempool() void RegisterMempoolRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ - // category actor (function) - // -------- ---------------- {"rawtransactions", &sendrawtransaction}, {"rawtransactions", &testmempoolaccept}, {"blockchain", &getmempoolancestors}, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 211026c8d9..e4b38efe07 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -1249,30 +1249,25 @@ static RPCHelpMan estimaterawfee() }; } -void RegisterMiningRPCCommands(CRPCTable &t) +void RegisterMiningRPCCommands(CRPCTable& t) { -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // --------------------- ----------------------- - { "mining", &getnetworkhashps, }, - { "mining", &getmininginfo, }, - { "mining", &prioritisetransaction, }, - { "mining", &getblocktemplate, }, - { "mining", &submitblock, }, - { "mining", &submitheader, }, - - - { "hidden", &generatetoaddress, }, - { "hidden", &generatetodescriptor, }, - { "hidden", &generateblock, }, - - { "util", &estimatesmartfee, }, - - { "hidden", &estimaterawfee, }, - { "hidden", &generate, }, -}; -// clang-format on + static const CRPCCommand commands[]{ + {"mining", &getnetworkhashps}, + {"mining", &getmininginfo}, + {"mining", &prioritisetransaction}, + {"mining", &getblocktemplate}, + {"mining", &submitblock}, + {"mining", &submitheader}, + + {"hidden", &generatetoaddress}, + {"hidden", &generatetodescriptor}, + {"hidden", &generateblock}, + + {"util", &estimatesmartfee}, + + {"hidden", &estimaterawfee}, + {"hidden", &generate}, + }; for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 99671ee6ac..98d751e69d 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -484,9 +484,8 @@ static RPCHelpMan mockscheduler() throw std::runtime_error("delta_time must be between 1 and 3600 seconds (1 hr)"); } - auto node_context = util::AnyPtr<NodeContext>(request.context); + auto node_context = CHECK_NONFATAL(util::AnyPtr<NodeContext>(request.context)); // protect against null pointer dereference - CHECK_NONFATAL(node_context); CHECK_NONFATAL(node_context->scheduler); node_context->scheduler->MockForward(std::chrono::seconds(delta_seconds)); @@ -791,33 +790,27 @@ static RPCHelpMan getindexinfo() }; } -void RegisterMiscRPCCommands(CRPCTable &t) +void RegisterMiscRPCCommands(CRPCTable& t) { -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // --------------------- ------------------------ - { "control", &getmemoryinfo, }, - { "control", &logging, }, - { "util", &validateaddress, }, - { "util", &createmultisig, }, - { "util", &deriveaddresses, }, - { "util", &getdescriptorinfo, }, - { "util", &verifymessage, }, - { "util", &signmessagewithprivkey, }, - { "util", &getindexinfo, }, - - /* Not shown in help */ - { "hidden", &setmocktime, }, - { "hidden", &mockscheduler, }, - { "hidden", &echo, }, - { "hidden", &echojson, }, - { "hidden", &echoipc, }, + static const CRPCCommand commands[]{ + {"control", &getmemoryinfo}, + {"control", &logging}, + {"util", &validateaddress}, + {"util", &createmultisig}, + {"util", &deriveaddresses}, + {"util", &getdescriptorinfo}, + {"util", &verifymessage}, + {"util", &signmessagewithprivkey}, + {"util", &getindexinfo}, + {"hidden", &setmocktime}, + {"hidden", &mockscheduler}, + {"hidden", &echo}, + {"hidden", &echojson}, + {"hidden", &echoipc}, #if defined(USE_SYSCALL_SANDBOX) - { "hidden", &invokedisallowedsyscall, }, + {"hidden", &invokedisallowedsyscall}, #endif // USE_SYSCALL_SANDBOX -}; -// clang-format on + }; for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 225feabf14..a9f9cb0153 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -959,30 +959,25 @@ static RPCHelpMan addpeeraddress() }; } -void RegisterNetRPCCommands(CRPCTable &t) -{ -// clang-format off -static const CRPCCommand commands[] = -{ // category actor - // --------------------- ----------------------- - { "network", &getconnectioncount, }, - { "network", &ping, }, - { "network", &getpeerinfo, }, - { "network", &addnode, }, - { "network", &disconnectnode, }, - { "network", &getaddednodeinfo, }, - { "network", &getnettotals, }, - { "network", &getnetworkinfo, }, - { "network", &setban, }, - { "network", &listbanned, }, - { "network", &clearbanned, }, - { "network", &setnetworkactive, }, - { "network", &getnodeaddresses, }, - - { "hidden", &addconnection, }, - { "hidden", &addpeeraddress, }, -}; -// clang-format on +void RegisterNetRPCCommands(CRPCTable& t) +{ + static const CRPCCommand commands[]{ + {"network", &getconnectioncount}, + {"network", &ping}, + {"network", &getpeerinfo}, + {"network", &addnode}, + {"network", &disconnectnode}, + {"network", &getaddednodeinfo}, + {"network", &getnettotals}, + {"network", &getnetworkinfo}, + {"network", &setban}, + {"network", &listbanned}, + {"network", &clearbanned}, + {"network", &setnetworkactive}, + {"network", &getnodeaddresses}, + {"hidden", &addconnection}, + {"hidden", &addpeeraddress}, + }; for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 8e4b396da9..ce8f16540f 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -33,6 +33,7 @@ #include <script/standard.h> #include <uint256.h> #include <util/bip32.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/string.h> #include <util/vector.h> @@ -466,7 +467,7 @@ static RPCHelpMan decodescript() // Should not be wrapped return false; } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); }()}; if (can_wrap_P2WSH) { UniValue sr(UniValue::VOBJ); @@ -1679,28 +1680,24 @@ static RPCHelpMan analyzepsbt() }; } -void RegisterRawTransactionRPCCommands(CRPCTable &t) +void RegisterRawTransactionRPCCommands(CRPCTable& t) { -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // --------------------- ----------------------- - { "rawtransactions", &getrawtransaction, }, - { "rawtransactions", &createrawtransaction, }, - { "rawtransactions", &decoderawtransaction, }, - { "rawtransactions", &decodescript, }, - { "rawtransactions", &combinerawtransaction, }, - { "rawtransactions", &signrawtransactionwithkey, }, - { "rawtransactions", &decodepsbt, }, - { "rawtransactions", &combinepsbt, }, - { "rawtransactions", &finalizepsbt, }, - { "rawtransactions", &createpsbt, }, - { "rawtransactions", &converttopsbt, }, - { "rawtransactions", &utxoupdatepsbt, }, - { "rawtransactions", &joinpsbts, }, - { "rawtransactions", &analyzepsbt, }, -}; -// clang-format on + static const CRPCCommand commands[]{ + {"rawtransactions", &getrawtransaction}, + {"rawtransactions", &createrawtransaction}, + {"rawtransactions", &decoderawtransaction}, + {"rawtransactions", &decodescript}, + {"rawtransactions", &combinerawtransaction}, + {"rawtransactions", &signrawtransactionwithkey}, + {"rawtransactions", &decodepsbt}, + {"rawtransactions", &combinepsbt}, + {"rawtransactions", &finalizepsbt}, + {"rawtransactions", &createpsbt}, + {"rawtransactions", &converttopsbt}, + {"rawtransactions", &utxoupdatepsbt}, + {"rawtransactions", &joinpsbts}, + {"rawtransactions", &analyzepsbt}, + }; for (const auto& c : commands) { t.appendCommand(c.name, &c); } diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index 95a7c25b93..d0e068de19 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -82,7 +82,7 @@ bool GenerateAuthCookie(std::string *cookie_out) { const size_t COOKIE_SIZE = 32; unsigned char rand_pwd[COOKIE_SIZE]; - GetRandBytes(rand_pwd, COOKIE_SIZE); + GetRandBytes(rand_pwd); std::string cookie = COOKIEAUTH_USER + ":" + HexStr(rand_pwd); /** the umask determines what permissions are used to create this file - diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 333ed6f5da..3817a4a45c 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -9,10 +9,9 @@ #include <shutdown.h> #include <sync.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/system.h> -#include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/split.hpp> #include <boost/signals2/signal.hpp> #include <cassert> @@ -248,17 +247,13 @@ static RPCHelpMan getrpcinfo() }; } -// clang-format off -static const CRPCCommand vRPCCommands[] = -{ // category actor (function) - // --------------------- ----------------------- +static const CRPCCommand vRPCCommands[]{ /* Overall control/query calls */ - { "control", &getrpcinfo, }, - { "control", &help, }, - { "control", &stop, }, - { "control", &uptime, }, + {"control", &getrpcinfo}, + {"control", &help}, + {"control", &stop}, + {"control", &uptime}, }; -// clang-format on CRPCTable::CRPCTable() { @@ -407,8 +402,7 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c // Process expected parameters. int hole = 0; for (const std::string &argNamePattern: argNames) { - std::vector<std::string> vargNames; - boost::algorithm::split(vargNames, argNamePattern, boost::algorithm::is_any_of("|")); + std::vector<std::string> vargNames = SplitString(argNamePattern, '|'); auto fr = argsIn.end(); for (const std::string & argName : vargNames) { fr = argsIn.find(argName); diff --git a/src/rpc/txoutproof.cpp b/src/rpc/txoutproof.cpp index a5443b0329..d16820baeb 100644 --- a/src/rpc/txoutproof.cpp +++ b/src/rpc/txoutproof.cpp @@ -172,8 +172,6 @@ static RPCHelpMan verifytxoutproof() void RegisterTxoutProofRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ - // category actor (function) - // -------- ---------------- {"blockchain", &gettxoutproof}, {"blockchain", &verifytxoutproof}, }; diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 01fae140cc..ef3e58494e 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -9,15 +9,13 @@ #include <script/descriptor.h> #include <script/signingprovider.h> #include <tinyformat.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> #include <tuple> -#include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/split.hpp> - const std::string UNIX_EPOCH_TIME = "UNIX epoch time"; const std::string EXAMPLE_ADDRESS[2] = {"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl", "bc1q02ad21edsxd23d32dfgqqsz4vv4nmtfzuklhy3"}; @@ -513,8 +511,7 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP { std::set<std::string> named_args; for (const auto& arg : m_args) { - std::vector<std::string> names; - boost::split(names, arg.m_names, boost::is_any_of("|")); + std::vector<std::string> names = SplitString(arg.m_names, '|'); // Should have unique named arguments for (const std::string& name : names) { CHECK_NONFATAL(named_args.insert(name).second); @@ -542,7 +539,7 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP // Null values are accepted in all arguments break; default: - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); break; } } @@ -665,8 +662,7 @@ UniValue RPCHelpMan::GetArgMap() const UniValue arr{UniValue::VARR}; for (int i{0}; i < int(m_args.size()); ++i) { const auto& arg = m_args.at(i); - std::vector<std::string> arg_names; - boost::split(arg_names, arg.m_names, boost::is_any_of("|")); + std::vector<std::string> arg_names = SplitString(arg.m_names, '|'); for (const auto& arg_name : arg_names) { UniValue map{UniValue::VARR}; map.push_back(m_name); @@ -793,7 +789,7 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const return; } case Type::ANY: { - CHECK_NONFATAL(false); // Only for testing + NONFATAL_UNREACHABLE(); // Only for testing } case Type::NONE: { sections.PushSection({indent + "null" + maybe_separator, Description("json null")}); @@ -860,7 +856,7 @@ void RPCResult::ToSections(Sections& sections, const OuterType outer_type, const return; } } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } bool RPCResult::MatchesType(const UniValue& result) const @@ -938,7 +934,7 @@ bool RPCResult::MatchesType(const UniValue& result) const return true; } } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } void RPCResult::CheckInnerDoc() const @@ -984,9 +980,9 @@ std::string RPCArg::ToStringObj(const bool oneline) const case Type::OBJ: case Type::OBJ_USER_KEYS: // Currently unused, so avoid writing dead code - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } std::string RPCArg::ToString(const bool oneline) const @@ -1021,7 +1017,7 @@ std::string RPCArg::ToString(const bool oneline) const return "[" + res + "...]"; } } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } static std::pair<int64_t, int64_t> ParseRange(const UniValue& value) diff --git a/src/script/interpreter.h b/src/script/interpreter.h index fbd904fb7b..adf454aa15 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -151,7 +151,7 @@ bool CheckSignatureEncoding(const std::vector<unsigned char> &vchSig, unsigned i struct PrecomputedTransactionData { // BIP341 precomputed data. - // These are single-SHA256, see https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki#cite_note-15. + // These are single-SHA256, see https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki#cite_note-16. uint256 m_prevouts_single_hash; uint256 m_sequences_single_hash; uint256 m_outputs_single_hash; diff --git a/src/serialize.h b/src/serialize.h index 44bb471f25..a1cce78451 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -472,10 +472,10 @@ struct CustomUintFormatter if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); if (BigEndian) { uint64_t raw = htobe64(v); - s.write({BytePtr(&raw) + 8 - Bytes, Bytes}); + s.write({AsBytePtr(&raw) + 8 - Bytes, Bytes}); } else { uint64_t raw = htole64(v); - s.write({BytePtr(&raw), Bytes}); + s.write({AsBytePtr(&raw), Bytes}); } } @@ -485,10 +485,10 @@ struct CustomUintFormatter static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small"); uint64_t raw = 0; if (BigEndian) { - s.read({BytePtr(&raw) + 8 - Bytes, Bytes}); + s.read({AsBytePtr(&raw) + 8 - Bytes, Bytes}); v = static_cast<I>(be64toh(raw)); } else { - s.read({BytePtr(&raw), Bytes}); + s.read({AsBytePtr(&raw), Bytes}); v = static_cast<I>(le64toh(raw)); } } diff --git a/src/span.h b/src/span.h index b627b993c2..444d63001f 100644 --- a/src/span.h +++ b/src/span.h @@ -245,19 +245,19 @@ T& SpanPopBack(Span<T>& span) //! Convert a data pointer to a std::byte data pointer. //! Where possible, please use the safer AsBytes helpers. -inline const std::byte* BytePtr(const void* data) { return reinterpret_cast<const std::byte*>(data); } -inline std::byte* BytePtr(void* data) { return reinterpret_cast<std::byte*>(data); } +inline const std::byte* AsBytePtr(const void* data) { return reinterpret_cast<const std::byte*>(data); } +inline std::byte* AsBytePtr(void* data) { return reinterpret_cast<std::byte*>(data); } // From C++20 as_bytes and as_writeable_bytes template <typename T> Span<const std::byte> AsBytes(Span<T> s) noexcept { - return {BytePtr(s.data()), s.size_bytes()}; + return {AsBytePtr(s.data()), s.size_bytes()}; } template <typename T> Span<std::byte> AsWritableBytes(Span<T> s) noexcept { - return {BytePtr(s.data()), s.size_bytes()}; + return {AsBytePtr(s.data()), s.size_bytes()}; } template <typename V> diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index a9aa928082..c6bd685189 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -37,7 +37,7 @@ struct secure_allocator : public std::allocator<T> { typedef secure_allocator<_Other> other; }; - T* allocate(std::size_t n, const void* hint = 0) + T* allocate(std::size_t n, const void* hint = nullptr) { T* allocation = static_cast<T*>(LockedPoolManager::Instance().alloc(sizeof(T) * n)); if (!allocation) { diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index efc30b6822..12cf1176a6 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -23,7 +23,7 @@ using namespace std::literals; using node::NodeContext; -static const std::vector<bool> EMPTY_ASMAP; +static NetGroupManager EMPTY_NETGROUPMAN{std::vector<bool>()}; static const bool DETERMINISTIC{true}; static int32_t GetCheckRatio(const NodeContext& node_ctx) @@ -62,7 +62,7 @@ BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -96,7 +96,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) BOOST_CHECK(addrman->size() >= 1); // Test: reset addrman and test AddrMan::Add multiple addresses works as expected - addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); std::vector<CAddress> vAddr; vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); @@ -106,7 +106,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) BOOST_AUTO_TEST_CASE(addrman_ports) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -135,7 +135,7 @@ BOOST_AUTO_TEST_CASE(addrman_ports) BOOST_AUTO_TEST_CASE(addrman_select) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -194,7 +194,7 @@ BOOST_AUTO_TEST_CASE(addrman_select) BOOST_AUTO_TEST_CASE(addrman_new_collisions) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -223,7 +223,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) BOOST_AUTO_TEST_CASE(addrman_new_multiplicity) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CAddress addr{CAddress(ResolveService("253.3.3.3", 8333), NODE_NONE)}; int64_t start_time{GetAdjustedTime()}; addr.nTime = start_time; @@ -255,7 +255,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_multiplicity) BOOST_AUTO_TEST_CASE(addrman_tried_collisions) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source = ResolveIP("252.2.2.2"); @@ -286,7 +286,7 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) BOOST_AUTO_TEST_CASE(addrman_getaddr) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); // Test: Sanity check, GetAddr should never return anything if addrman // is empty. @@ -357,27 +357,25 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - std::vector<bool> asmap; // use /16 - - BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN), 40); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN) != info1.GetTriedBucket(nKey2, EMPTY_NETGROUPMAN)); // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN) != info2.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN)); std::set<int> buckets; for (int i = 0; i < 255; i++) { AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); - int bucket = infoi.GetTriedBucket(nKey1, asmap); + int bucket = infoi.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN); buckets.insert(bucket); } // Test: IP addresses in the same /16 prefix should @@ -389,7 +387,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) AddrInfo infoj = AddrInfo( CAddress(ResolveService("250." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("250." + ToString(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1, asmap); + int bucket = infoj.GetTriedBucket(nKey1, EMPTY_NETGROUPMAN); buckets.insert(bucket); } // Test: IP addresses in the different /16 prefix should map to more than @@ -409,27 +407,25 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - std::vector<bool> asmap; // use /16 - // Test: Make sure the buckets are what we expect - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 786); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, EMPTY_NETGROUPMAN), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, EMPTY_NETGROUPMAN), 786); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); + BOOST_CHECK(info1.GetNewBucket(nKey1, EMPTY_NETGROUPMAN) != info1.GetNewBucket(nKey2, EMPTY_NETGROUPMAN)); // Test: Ports should not affect bucket placement in the addr AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, EMPTY_NETGROUPMAN), info2.GetNewBucket(nKey1, EMPTY_NETGROUPMAN)); std::set<int> buckets; for (int i = 0; i < 255; i++) { AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); - int bucket = infoi.GetNewBucket(nKey1, asmap); + int bucket = infoi.GetNewBucket(nKey1, EMPTY_NETGROUPMAN); buckets.insert(bucket); } // Test: IP addresses in the same group (\16 prefix for IPv4) should @@ -442,7 +438,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) ResolveService( ToString(250 + (j / 255)) + "." + ToString(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); - int bucket = infoj.GetNewBucket(nKey1, asmap); + int bucket = infoj.GetNewBucket(nKey1, EMPTY_NETGROUPMAN); buckets.insert(bucket); } // Test: IP addresses in the same source groups should map to NO MORE @@ -454,7 +450,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + ToString(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1, asmap); + int bucket = infoj.GetNewBucket(nKey1, EMPTY_NETGROUPMAN); buckets.insert(bucket); } // Test: IP addresses in the different source groups should map to MORE @@ -475,6 +471,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) // 101.8.0.0/16 AS8 BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) { + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + NetGroupManager ngm_asmap{asmap}; + CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); @@ -486,27 +485,25 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); - - BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 236); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, ngm_asmap), 236); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, ngm_asmap) != info1.GetTriedBucket(nKey2, ngm_asmap)); // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, ngm_asmap) != info2.GetTriedBucket(nKey1, ngm_asmap)); std::set<int> buckets; for (int j = 0; j < 255; j++) { AddrInfo infoj = AddrInfo( CAddress(ResolveService("101." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("101." + ToString(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1, asmap); + int bucket = infoj.GetTriedBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the different /16 prefix MAY map to more than @@ -518,7 +515,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) AddrInfo infoj = AddrInfo( CAddress(ResolveService("250." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("250." + ToString(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1, asmap); + int bucket = infoj.GetTriedBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the different /16 prefix MAY NOT map to more than @@ -528,6 +525,9 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) { + std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + NetGroupManager ngm_asmap{asmap}; + CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); @@ -538,27 +538,25 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); - std::vector<bool> asmap = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); - // Test: Make sure the buckets are what we expect - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 795); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 795); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, ngm_asmap), 795); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, ngm_asmap), 795); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); + BOOST_CHECK(info1.GetNewBucket(nKey1, ngm_asmap) != info1.GetNewBucket(nKey2, ngm_asmap)); // Test: Ports should not affect bucket placement in the addr AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, ngm_asmap), info2.GetNewBucket(nKey1, ngm_asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); - int bucket = infoi.GetNewBucket(nKey1, asmap); + int bucket = infoi.GetNewBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the same /16 prefix @@ -571,7 +569,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) ResolveService( ToString(250 + (j / 255)) + "." + ToString(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); - int bucket = infoj.GetNewBucket(nKey1, asmap); + int bucket = infoj.GetNewBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the same source /16 prefix should not map to more @@ -583,7 +581,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("101." + ToString(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1, asmap); + int bucket = infoj.GetNewBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the different source /16 prefixes usually map to MORE @@ -595,7 +593,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + ToString(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1, asmap); + int bucket = infoj.GetNewBucket(nKey1, ngm_asmap); buckets.insert(bucket); } // Test: IP addresses in the different source /16 prefixes sometimes map to NO MORE @@ -606,11 +604,12 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) BOOST_AUTO_TEST_CASE(addrman_serialization) { std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); + NetGroupManager netgroupman{asmap1}; const auto ratio = GetCheckRatio(m_node); - auto addrman_asmap1 = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); - auto addrman_asmap1_dup = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); - auto addrman_noasmap = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, ratio); + auto addrman_asmap1 = std::make_unique<AddrMan>(netgroupman, DETERMINISTIC, ratio); + auto addrman_asmap1_dup = std::make_unique<AddrMan>(netgroupman, DETERMINISTIC, ratio); + auto addrman_noasmap = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, ratio); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); @@ -639,8 +638,8 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(addr_pos1.position != addr_pos3.position); // deserializing non-asmaped peers.dat to asmaped addrman - addrman_asmap1 = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); - addrman_noasmap = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, ratio); + addrman_asmap1 = std::make_unique<AddrMan>(netgroupman, DETERMINISTIC, ratio); + addrman_noasmap = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, ratio); addrman_noasmap->Add({addr}, default_source); stream << *addrman_noasmap; stream >> *addrman_asmap1; @@ -651,8 +650,8 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(addr_pos4 == addr_pos2); // used to map to different buckets, now maps to the same bucket. - addrman_asmap1 = std::make_unique<AddrMan>(asmap1, DETERMINISTIC, ratio); - addrman_noasmap = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, ratio); + addrman_asmap1 = std::make_unique<AddrMan>(netgroupman, DETERMINISTIC, ratio); + addrman_noasmap = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, ratio); CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); addrman_noasmap->Add({addr, addr2}, default_source); @@ -671,7 +670,7 @@ BOOST_AUTO_TEST_CASE(remove_invalid) { // Confirm that invalid addresses are ignored in unserialization. - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); const CAddress new1{ResolveService("5.5.5.5"), NODE_NONE}; @@ -703,14 +702,14 @@ BOOST_AUTO_TEST_CASE(remove_invalid) BOOST_REQUIRE(pos + sizeof(tried2_raw_replacement) <= stream.size()); memcpy(stream.data() + pos, tried2_raw_replacement, sizeof(tried2_raw_replacement)); - addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); stream >> *addrman; BOOST_CHECK_EQUAL(addrman->size(), 2); } BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); BOOST_CHECK(addrman->size() == 0); @@ -743,7 +742,7 @@ BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) BOOST_AUTO_TEST_CASE(addrman_noevict) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); // Add 35 addresses. CNetAddr source = ResolveIP("252.2.2.2"); @@ -795,7 +794,7 @@ BOOST_AUTO_TEST_CASE(addrman_noevict) BOOST_AUTO_TEST_CASE(addrman_evictionworks) { - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); BOOST_CHECK(addrman->size() == 0); @@ -865,7 +864,7 @@ static CDataStream AddrmanToStream(const AddrMan& addrman) BOOST_AUTO_TEST_CASE(load_addrman) { - AddrMan addrman{EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)}; + AddrMan addrman{EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)}; CService addr1, addr2, addr3; BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); @@ -884,7 +883,7 @@ BOOST_AUTO_TEST_CASE(load_addrman) // Test that the de-serialization does not throw an exception. CDataStream ssPeers1 = AddrmanToStream(addrman); bool exceptionThrown = false; - AddrMan addrman1{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; + AddrMan addrman1{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman1.size() == 0); try { @@ -901,7 +900,7 @@ BOOST_AUTO_TEST_CASE(load_addrman) // Test that ReadFromStream creates an addrman with the correct number of addrs. CDataStream ssPeers2 = AddrmanToStream(addrman); - AddrMan addrman2{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; + AddrMan addrman2{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman2.size() == 0); ReadFromStream(addrman2, ssPeers2); BOOST_CHECK(addrman2.size() == 3); @@ -939,7 +938,7 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) // Test that the de-serialization of corrupted peers.dat throws an exception. CDataStream ssPeers1 = MakeCorruptPeersDat(); bool exceptionThrown = false; - AddrMan addrman1{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; + AddrMan addrman1{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman1.size() == 0); try { unsigned char pchMsgTmp[4]; @@ -955,7 +954,7 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) // Test that ReadFromStream fails if peers.dat is corrupt CDataStream ssPeers2 = MakeCorruptPeersDat(); - AddrMan addrman2{EMPTY_ASMAP, !DETERMINISTIC, GetCheckRatio(m_node)}; + AddrMan addrman2{EMPTY_NETGROUPMAN, !DETERMINISTIC, GetCheckRatio(m_node)}; BOOST_CHECK(addrman2.size() == 0); BOOST_CHECK_THROW(ReadFromStream(addrman2, ssPeers2), std::ios_base::failure); } @@ -963,7 +962,7 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) BOOST_AUTO_TEST_CASE(addrman_update_address) { // Tests updating nTime via Connected() and nServices via SetServices() - auto addrman = std::make_unique<AddrMan>(EMPTY_ASMAP, DETERMINISTIC, GetCheckRatio(m_node)); + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); CNetAddr source{ResolveIP("252.2.2.2")}; CAddress addr{CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE)}; diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index 5fab7f0d1e..c6109dfeb0 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -22,20 +22,16 @@ BOOST_AUTO_TEST_CASE(base32_testvectors) BOOST_CHECK_EQUAL(strEnc, vstrOut[i]); strEnc = EncodeBase32(vstrIn[i], false); BOOST_CHECK_EQUAL(strEnc, vstrOutNoPadding[i]); - std::string strDec = DecodeBase32(vstrOut[i]); - BOOST_CHECK_EQUAL(strDec, vstrIn[i]); + auto dec = DecodeBase32(vstrOut[i]); + BOOST_REQUIRE(dec); + BOOST_CHECK_MESSAGE(MakeByteSpan(*dec) == MakeByteSpan(vstrIn[i]), vstrOut[i]); } // Decoding strings with embedded NUL characters should fail - bool failure; - (void)DecodeBase32("invalid\0"s, &failure); // correct size, invalid due to \0 - BOOST_CHECK(failure); - (void)DecodeBase32("AWSX3VPP"s, &failure); // valid - BOOST_CHECK(!failure); - (void)DecodeBase32("AWSX3VPP\0invalid"s, &failure); // correct size, invalid due to \0 - BOOST_CHECK(failure); - (void)DecodeBase32("AWSX3VPPinvalid"s, &failure); // invalid size - BOOST_CHECK(failure); + BOOST_CHECK(!DecodeBase32("invalid\0"s)); // correct size, invalid due to \0 + BOOST_CHECK(DecodeBase32("AWSX3VPP"s)); // valid + BOOST_CHECK(!DecodeBase32("AWSX3VPP\0invalid"s)); // correct size, invalid due to \0 + BOOST_CHECK(!DecodeBase32("AWSX3VPPinvalid"s)); // invalid size } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 6ee1b83691..54a02c6bf8 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -19,8 +19,9 @@ BOOST_AUTO_TEST_CASE(base64_testvectors) { std::string strEnc = EncodeBase64(vstrIn[i]); BOOST_CHECK_EQUAL(strEnc, vstrOut[i]); - std::string strDec = DecodeBase64(strEnc); - BOOST_CHECK_EQUAL(strDec, vstrIn[i]); + auto dec = DecodeBase64(strEnc); + BOOST_REQUIRE(dec); + BOOST_CHECK_MESSAGE(MakeByteSpan(*dec) == MakeByteSpan(vstrIn[i]), vstrOut[i]); } { @@ -34,15 +35,10 @@ BOOST_AUTO_TEST_CASE(base64_testvectors) } // Decoding strings with embedded NUL characters should fail - bool failure; - (void)DecodeBase64("invalid\0"s, &failure); - BOOST_CHECK(failure); - (void)DecodeBase64("nQB/pZw="s, &failure); - BOOST_CHECK(!failure); - (void)DecodeBase64("nQB/pZw=\0invalid"s, &failure); - BOOST_CHECK(failure); - (void)DecodeBase64("nQB/pZw=invalid\0"s, &failure); - BOOST_CHECK(failure); + BOOST_CHECK(!DecodeBase64("invalid\0"s)); + BOOST_CHECK(DecodeBase64("nQB/pZw="s)); + BOOST_CHECK(!DecodeBase64("nQB/pZw=\0invalid"s)); + BOOST_CHECK(!DecodeBase64("nQB/pZw=invalid\0"s)); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index f03ff5ba3a..7a917649a8 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -4,7 +4,6 @@ // Unit tests for denial-of-service detection/prevention code -#include <arith_uint256.h> #include <banman.h> #include <chainparams.h> #include <net.h> @@ -16,7 +15,6 @@ #include <serialize.h> #include <test/util/net.h> #include <test/util/setup_common.h> -#include <txorphanage.h> #include <util/string.h> #include <util/system.h> #include <util/time.h> @@ -49,7 +47,7 @@ BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { const CChainParams& chainparams = Params(); - auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); // Disable inactivity checks for this test to avoid interference static_cast<ConnmanTestMsg*>(connman.get())->SetPeerConnectTimeout(99999s); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, @@ -139,7 +137,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { NodeId id{0}; const CChainParams& chainparams = Params(); - auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, false); @@ -217,7 +215,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) { NodeId id{0}; const CChainParams& chainparams = Params(); - auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, false); @@ -280,7 +278,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) { const CChainParams& chainparams = Params(); auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); - auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, false); @@ -396,7 +394,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) { const CChainParams& chainparams = Params(); auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); - auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); + auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, false); @@ -430,121 +428,4 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) peerLogic->FinalizeNode(dummyNode); } -class TxOrphanageTest : public TxOrphanage -{ -public: - inline size_t CountOrphans() const EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) - { - return m_orphans.size(); - } - - CTransactionRef RandomOrphan() EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) - { - std::map<uint256, OrphanTx>::iterator it; - it = m_orphans.lower_bound(InsecureRand256()); - if (it == m_orphans.end()) - it = m_orphans.begin(); - return it->second.tx; - } -}; - -static void MakeNewKeyWithFastRandomContext(CKey& key) -{ - std::vector<unsigned char> keydata; - keydata = g_insecure_rand_ctx.randbytes(32); - key.Set(keydata.data(), keydata.data() + keydata.size(), /*fCompressedIn=*/true); - assert(key.IsValid()); -} - -BOOST_AUTO_TEST_CASE(DoS_mapOrphans) -{ - // This test had non-deterministic coverage due to - // randomly selected seeds. - // This seed is chosen so that all branches of the function - // ecdsa_signature_parse_der_lax are executed during this test. - // Specifically branches that run only when an ECDSA - // signature's R and S values have leading zeros. - g_insecure_rand_ctx = FastRandomContext(ArithToUint256(arith_uint256(33))); - - TxOrphanageTest orphanage; - CKey key; - MakeNewKeyWithFastRandomContext(key); - FillableSigningProvider keystore; - BOOST_CHECK(keystore.AddKey(key)); - - LOCK(g_cs_orphans); - - // 50 orphan transactions: - for (int i = 0; i < 50; i++) - { - CMutableTransaction tx; - tx.vin.resize(1); - tx.vin[0].prevout.n = 0; - tx.vin[0].prevout.hash = InsecureRand256(); - tx.vin[0].scriptSig << OP_1; - tx.vout.resize(1); - tx.vout[0].nValue = 1*CENT; - tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); - - orphanage.AddTx(MakeTransactionRef(tx), i); - } - - // ... and 50 that depend on other orphans: - for (int i = 0; i < 50; i++) - { - CTransactionRef txPrev = orphanage.RandomOrphan(); - - CMutableTransaction tx; - tx.vin.resize(1); - tx.vin[0].prevout.n = 0; - tx.vin[0].prevout.hash = txPrev->GetHash(); - tx.vout.resize(1); - tx.vout[0].nValue = 1*CENT; - tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); - BOOST_CHECK(SignSignature(keystore, *txPrev, tx, 0, SIGHASH_ALL)); - - orphanage.AddTx(MakeTransactionRef(tx), i); - } - - // This really-big orphan should be ignored: - for (int i = 0; i < 10; i++) - { - CTransactionRef txPrev = orphanage.RandomOrphan(); - - CMutableTransaction tx; - tx.vout.resize(1); - tx.vout[0].nValue = 1*CENT; - tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); - tx.vin.resize(2777); - for (unsigned int j = 0; j < tx.vin.size(); j++) - { - tx.vin[j].prevout.n = j; - tx.vin[j].prevout.hash = txPrev->GetHash(); - } - BOOST_CHECK(SignSignature(keystore, *txPrev, tx, 0, SIGHASH_ALL)); - // Re-use same signature for other inputs - // (they don't have to be valid for this test) - for (unsigned int j = 1; j < tx.vin.size(); j++) - tx.vin[j].scriptSig = tx.vin[0].scriptSig; - - BOOST_CHECK(!orphanage.AddTx(MakeTransactionRef(tx), i)); - } - - // Test EraseOrphansFor: - for (NodeId i = 0; i < 3; i++) - { - size_t sizeBefore = orphanage.CountOrphans(); - orphanage.EraseForPeer(i); - BOOST_CHECK(orphanage.CountOrphans() < sizeBefore); - } - - // Test LimitOrphanTxSize() function: - orphanage.LimitOrphans(40); - BOOST_CHECK(orphanage.CountOrphans() <= 40); - orphanage.LimitOrphans(10); - BOOST_CHECK(orphanage.CountOrphans() <= 10); - orphanage.LimitOrphans(0); - BOOST_CHECK(orphanage.CountOrphans() == 0); -} - BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index ba917dec2a..af7a282781 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -37,11 +37,19 @@ void initialize_addrman() g_setup = testing_setup.get(); } +[[nodiscard]] inline NetGroupManager ConsumeNetGroupManager(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); + if (!SanityCheckASMap(asmap, 128)) asmap.clear(); + return NetGroupManager(asmap); +} + FUZZ_TARGET_INIT(data_stream_addr_man, initialize_addrman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; CDataStream data_stream = ConsumeDataStream(fuzzed_data_provider); - AddrMan addr_man{/*asmap=*/std::vector<bool>(), /*deterministic=*/false, GetCheckRatio()}; + NetGroupManager netgroupman{ConsumeNetGroupManager(fuzzed_data_provider)}; + AddrMan addr_man(netgroupman, /*deterministic=*/false, GetCheckRatio()); try { ReadFromStream(addr_man, data_stream); } catch (const std::exception&) { @@ -124,8 +132,8 @@ void FillAddrman(AddrMan& addrman, FuzzedDataProvider& fuzzed_data_provider) class AddrManDeterministic : public AddrMan { public: - explicit AddrManDeterministic(std::vector<bool> asmap, FuzzedDataProvider& fuzzed_data_provider) - : AddrMan{std::move(asmap), /*deterministic=*/true, GetCheckRatio()} + explicit AddrManDeterministic(const NetGroupManager& netgroupman, FuzzedDataProvider& fuzzed_data_provider) + : AddrMan(netgroupman, /*deterministic=*/true, GetCheckRatio()) { WITH_LOCK(m_impl->cs, m_impl->insecure_rand = FastRandomContext{ConsumeUInt256(fuzzed_data_provider)}); } @@ -223,19 +231,12 @@ public: } }; -[[nodiscard]] inline std::vector<bool> ConsumeAsmap(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); - if (!SanityCheckASMap(asmap, 128)) asmap.clear(); - return asmap; -} - FUZZ_TARGET_INIT(addrman, initialize_addrman) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - std::vector<bool> asmap = ConsumeAsmap(fuzzed_data_provider); - auto addr_man_ptr = std::make_unique<AddrManDeterministic>(asmap, fuzzed_data_provider); + NetGroupManager netgroupman{ConsumeNetGroupManager(fuzzed_data_provider)}; + auto addr_man_ptr = std::make_unique<AddrManDeterministic>(netgroupman, fuzzed_data_provider); if (fuzzed_data_provider.ConsumeBool()) { const std::vector<uint8_t> serialized_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; CDataStream ds(serialized_data, SER_DISK, INIT_PROTO_VERSION); @@ -244,7 +245,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) try { ds >> *addr_man_ptr; } catch (const std::ios_base::failure&) { - addr_man_ptr = std::make_unique<AddrManDeterministic>(asmap, fuzzed_data_provider); + addr_man_ptr = std::make_unique<AddrManDeterministic>(netgroupman, fuzzed_data_provider); } } AddrManDeterministic& addr_man = *addr_man_ptr; @@ -313,9 +314,9 @@ FUZZ_TARGET_INIT(addrman_serdeser, initialize_addrman) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - std::vector<bool> asmap = ConsumeAsmap(fuzzed_data_provider); - AddrManDeterministic addr_man1{asmap, fuzzed_data_provider}; - AddrManDeterministic addr_man2{asmap, fuzzed_data_provider}; + NetGroupManager netgroupman{ConsumeNetGroupManager(fuzzed_data_provider)}; + AddrManDeterministic addr_man1{netgroupman, fuzzed_data_provider}; + AddrManDeterministic addr_man2{netgroupman, fuzzed_data_provider}; CDataStream data_stream(SER_NETWORK, PROTOCOL_VERSION); diff --git a/src/test/fuzz/asmap.cpp b/src/test/fuzz/asmap.cpp index 95be963dc8..1720f8e0ab 100644 --- a/src/test/fuzz/asmap.cpp +++ b/src/test/fuzz/asmap.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <netaddress.h> +#include <netgroup.h> #include <test/fuzz/fuzz.h> #include <util/asmap.h> @@ -56,5 +57,6 @@ FUZZ_TARGET(asmap) memcpy(&ipv4, addr_data, addr_size); net_addr.SetIP(CNetAddr{ipv4}); } - (void)net_addr.GetMappedAS(asmap); + NetGroupManager netgroupman{asmap}; + (void)netgroupman.GetMappedAS(net_addr); } diff --git a/src/test/fuzz/base_encode_decode.cpp b/src/test/fuzz/base_encode_decode.cpp index 196410e29c..48356065b0 100644 --- a/src/test/fuzz/base_encode_decode.cpp +++ b/src/test/fuzz/base_encode_decode.cpp @@ -26,7 +26,7 @@ FUZZ_TARGET_INIT(base_encode_decode, initialize_base_encode_decode) std::vector<unsigned char> decoded; if (DecodeBase58(random_encoded_string, decoded, 100)) { const std::string encoded_string = EncodeBase58(decoded); - assert(encoded_string == TrimString(encoded_string)); + assert(encoded_string == TrimStringView(encoded_string)); assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); } @@ -36,17 +36,16 @@ FUZZ_TARGET_INIT(base_encode_decode, initialize_base_encode_decode) assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); } - bool pf_invalid; - std::string decoded_string = DecodeBase32(random_encoded_string, &pf_invalid); - if (!pf_invalid) { - const std::string encoded_string = EncodeBase32(decoded_string); - assert(encoded_string == TrimString(encoded_string)); + auto result = DecodeBase32(random_encoded_string); + if (result) { + const std::string encoded_string = EncodeBase32(*result); + assert(encoded_string == TrimStringView(encoded_string)); assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); } - decoded_string = DecodeBase64(random_encoded_string, &pf_invalid); - if (!pf_invalid) { - const std::string encoded_string = EncodeBase64(decoded_string); + result = DecodeBase64(random_encoded_string); + if (result) { + const std::string encoded_string = EncodeBase64(*result); assert(encoded_string == TrimString(encoded_string)); assert(ToLower(encoded_string) == ToLower(TrimString(random_encoded_string))); } diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index a14d28f4ef..4406779015 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -19,12 +19,12 @@ #include <vector> namespace { -const BasicTestingSetup* g_setup; +const TestingSetup* g_setup; } // namespace void initialize_connman() { - static const auto testing_setup = MakeNoLogFileContext<>(); + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); g_setup = testing_setup.get(); } @@ -32,10 +32,11 @@ FUZZ_TARGET_INIT(connman, initialize_connman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); - AddrMan addrman(/*asmap=*/std::vector<bool>(), - /*deterministic=*/false, - g_setup->m_node.args->GetIntArg("-checkaddrman", 0)); - CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>(), addrman, fuzzed_data_provider.ConsumeBool()}; + CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + *g_setup->m_node.addrman, + *g_setup->m_node.netgroupman, + fuzzed_data_provider.ConsumeBool()}; CNetAddr random_netaddr; CNode random_node = ConsumeNode(fuzzed_data_provider); CSubNet random_subnet; diff --git a/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp b/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp index fcc96c6418..1b89d55773 100644 --- a/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp +++ b/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp @@ -128,7 +128,7 @@ void ECRYPT_encrypt_bytes(ECRYPT_ctx* x, const u8* m, u8* c, u32 bytes) { u32 x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; u32 j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; - u8* ctarget = NULL; + u8* ctarget = nullptr; u8 tmp[64]; uint32_t i; diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index ed6f172a2a..0a7d0c55bd 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -15,6 +15,7 @@ #include <merkleblock.h> #include <net.h> #include <netbase.h> +#include <netgroup.h> #include <node/utxo_snapshot.h> #include <primitives/block.h> #include <protocol.h> @@ -200,7 +201,8 @@ FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { BlockMerkleRoot(block, &mutated); }) FUZZ_TARGET_DESERIALIZE(addrman_deserialize, { - AddrMan am(/*asmap=*/std::vector<bool>(), + NetGroupManager netgroupman{std::vector<bool>()}; + AddrMan am(netgroupman, /*deterministic=*/false, g_setup->m_node.args->GetIntArg("-checkaddrman", 0)); DeserializeFromFuzzingInput(buffer, am); diff --git a/src/test/fuzz/http_request.cpp b/src/test/fuzz/http_request.cpp index 916e90e986..0fe18abaa9 100644 --- a/src/test/fuzz/http_request.cpp +++ b/src/test/fuzz/http_request.cpp @@ -39,7 +39,7 @@ FUZZ_TARGET(http_request) // and is a consequence of our hacky but necessary use of the internal function evhttp_parse_firstline_ in // this fuzzing harness. The workaround is not aesthetically pleasing, but it successfully avoids the troublesome // code path. " http:// HTTP/1.1\n" was a crashing input prior to this workaround. - const std::string http_buffer_str = ToLower({http_buffer.begin(), http_buffer.end()}); + const std::string http_buffer_str = ToLower(std::string{http_buffer.begin(), http_buffer.end()}); if (http_buffer_str.find(" http://") != std::string::npos || http_buffer_str.find(" https://") != std::string::npos || evhttp_parse_firstline_(evreq, evbuf) != 1 || evhttp_parse_headers_(evreq, evbuf) != 1) { evbuffer_free(evbuf); diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index 669688a80d..baa64bba0f 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -32,7 +32,8 @@ FUZZ_TARGET_INIT(psbt, initialize_psbt) FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; PartiallySignedTransaction psbt_mut; std::string error; - if (!DecodeRawPSBT(psbt_mut, fuzzed_data_provider.ConsumeRandomLengthString(), error)) { + auto str = fuzzed_data_provider.ConsumeRandomLengthString(); + if (!DecodeRawPSBT(psbt_mut, MakeByteSpan(str), error)) { return; } const PartiallySignedTransaction psbt = psbt_mut; @@ -79,7 +80,8 @@ FUZZ_TARGET_INIT(psbt, initialize_psbt) } PartiallySignedTransaction psbt_merge; - if (!DecodeRawPSBT(psbt_merge, fuzzed_data_provider.ConsumeRandomLengthString(), error)) { + str = fuzzed_data_provider.ConsumeRandomLengthString(); + if (!DecodeRawPSBT(psbt_merge, MakeByteSpan(str), error)) { psbt_merge = psbt; } psbt_mut = psbt; diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 00a3bed12f..a18a0de47e 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -11,8 +11,8 @@ #include <streams.h> #include <univalue.h> #include <util/strencodings.h> +#include <util/string.h> -#include <boost/algorithm/string.hpp> #include <cstdint> #include <string> #include <vector> @@ -130,8 +130,7 @@ unsigned int ParseScriptFlags(const std::string& str) if (str.empty()) return 0; unsigned int flags = 0; - std::vector<std::string> words; - boost::algorithm::split(words, str, boost::algorithm::is_any_of(",")); + std::vector<std::string> words = SplitString(str, ','); for (const std::string& word : words) { auto it = FLAG_NAMES.find(word); diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index ca57af25c4..e6064d19b6 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -42,7 +42,7 @@ bool LegacyParsePrechecks(const std::string& str) return false; if (str.size() >= 1 && (IsSpace(str[0]) || IsSpace(str[str.size() - 1]))) // No padding allowed return false; - if (!ValidAsCString(str)) // No embedded NUL characters allowed + if (!ContainsNoNUL(str)) // No embedded NUL characters allowed return false; return true; } @@ -188,7 +188,7 @@ FUZZ_TARGET(string) (void)TrimString(random_string_1); (void)TrimString(random_string_1, random_string_2); (void)urlDecode(random_string_1); - (void)ValidAsCString(random_string_1); + (void)ContainsNoNUL(random_string_1); (void)_(random_string_1.c_str()); try { throw scriptnum_error{random_string_1}; @@ -224,6 +224,7 @@ FUZZ_TARGET(string) int64_t amount_out; (void)ParseFixedPoint(random_string_1, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1024), &amount_out); } + (void)SplitString(random_string_1, fuzzed_data_provider.ConsumeIntegral<char>()); { (void)Untranslated(random_string_1); const bilingual_str bs1{random_string_1, random_string_2}; diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 61d334ab18..8cb0515a8a 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -204,7 +204,7 @@ BOOST_AUTO_TEST_CASE(key_key_negation) // create a dummy hash for signature comparison unsigned char rnd[8]; std::string str = "Bitcoin key verification\n"; - GetRandBytes(rnd, sizeof(rnd)); + GetRandBytes(rnd); uint256 hash; CHash256().Write(MakeUCharSpan(str)).Write(rnd).Finalize(hash); diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 8e6e911ec2..224dc88d0f 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -5,6 +5,7 @@ #include <net_permissions.h> #include <netaddress.h> #include <netbase.h> +#include <netgroup.h> #include <protocol.h> #include <serialize.h> #include <streams.h> @@ -315,22 +316,22 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_AUTO_TEST_CASE(netbase_getgroup) { - std::vector<bool> asmap; // use /16 - BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // Local -> !Routable() - BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // !Valid -> !Routable() - BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() - BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup(asmap) == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() - BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 - BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 - BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 - BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 - BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 - BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net - BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 + NetGroupManager netgroupman{std::vector<bool>()}; // use /16 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("127.0.0.1")) == std::vector<unsigned char>({0})); // Local -> !Routable() + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("257.0.0.1")) == std::vector<unsigned char>({0})); // !Valid -> !Routable() + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("10.0.0.1")) == std::vector<unsigned char>({0})); // RFC1918 -> !Routable() + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("169.254.1.1")) == std::vector<unsigned char>({0})); // RFC3927 -> !Routable() + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("1.2.3.4")) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // IPv4 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("::FFFF:0:102:304")) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("64:FF9B::102:304")) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("2002:102:304:9999:9999:9999:9999:9999")) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB")) == std::vector<unsigned char>({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999")) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net + BOOST_CHECK(netgroupman.GetGroup(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999")) == std::vector<unsigned char>({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 // baz.net sha256 hash: 12929400eb4607c4ac075f087167e75286b179c693eb059a01774b864e8fe505 std::vector<unsigned char> internal_group = {NET_INTERNAL, 0x12, 0x92, 0x94, 0x00, 0xeb, 0x46, 0x07, 0xc4, 0xac, 0x07}; - BOOST_CHECK(CreateInternal("baz.net").GetGroup(asmap) == internal_group); + BOOST_CHECK(netgroupman.GetGroup(CreateInternal("baz.net")) == internal_group); } BOOST_AUTO_TEST_CASE(netbase_parsenetwork) diff --git a/src/test/orphanage_tests.cpp b/src/test/orphanage_tests.cpp new file mode 100644 index 0000000000..842daa8bd4 --- /dev/null +++ b/src/test/orphanage_tests.cpp @@ -0,0 +1,137 @@ +// Copyright (c) 2011-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <arith_uint256.h> +#include <pubkey.h> +#include <script/sign.h> +#include <script/signingprovider.h> +#include <script/standard.h> +#include <test/util/setup_common.h> +#include <txorphanage.h> + +#include <array> +#include <cstdint> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(orphanage_tests, TestingSetup) + +class TxOrphanageTest : public TxOrphanage +{ +public: + inline size_t CountOrphans() const EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) + { + return m_orphans.size(); + } + + CTransactionRef RandomOrphan() EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) + { + std::map<uint256, OrphanTx>::iterator it; + it = m_orphans.lower_bound(InsecureRand256()); + if (it == m_orphans.end()) + it = m_orphans.begin(); + return it->second.tx; + } +}; + +static void MakeNewKeyWithFastRandomContext(CKey& key) +{ + std::vector<unsigned char> keydata; + keydata = g_insecure_rand_ctx.randbytes(32); + key.Set(keydata.data(), keydata.data() + keydata.size(), /*fCompressedIn=*/true); + assert(key.IsValid()); +} + +BOOST_AUTO_TEST_CASE(DoS_mapOrphans) +{ + // This test had non-deterministic coverage due to + // randomly selected seeds. + // This seed is chosen so that all branches of the function + // ecdsa_signature_parse_der_lax are executed during this test. + // Specifically branches that run only when an ECDSA + // signature's R and S values have leading zeros. + g_insecure_rand_ctx = FastRandomContext{uint256{33}}; + + TxOrphanageTest orphanage; + CKey key; + MakeNewKeyWithFastRandomContext(key); + FillableSigningProvider keystore; + BOOST_CHECK(keystore.AddKey(key)); + + LOCK(g_cs_orphans); + + // 50 orphan transactions: + for (int i = 0; i < 50; i++) + { + CMutableTransaction tx; + tx.vin.resize(1); + tx.vin[0].prevout.n = 0; + tx.vin[0].prevout.hash = InsecureRand256(); + tx.vin[0].scriptSig << OP_1; + tx.vout.resize(1); + tx.vout[0].nValue = 1*CENT; + tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); + + orphanage.AddTx(MakeTransactionRef(tx), i); + } + + // ... and 50 that depend on other orphans: + for (int i = 0; i < 50; i++) + { + CTransactionRef txPrev = orphanage.RandomOrphan(); + + CMutableTransaction tx; + tx.vin.resize(1); + tx.vin[0].prevout.n = 0; + tx.vin[0].prevout.hash = txPrev->GetHash(); + tx.vout.resize(1); + tx.vout[0].nValue = 1*CENT; + tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); + BOOST_CHECK(SignSignature(keystore, *txPrev, tx, 0, SIGHASH_ALL)); + + orphanage.AddTx(MakeTransactionRef(tx), i); + } + + // This really-big orphan should be ignored: + for (int i = 0; i < 10; i++) + { + CTransactionRef txPrev = orphanage.RandomOrphan(); + + CMutableTransaction tx; + tx.vout.resize(1); + tx.vout[0].nValue = 1*CENT; + tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); + tx.vin.resize(2777); + for (unsigned int j = 0; j < tx.vin.size(); j++) + { + tx.vin[j].prevout.n = j; + tx.vin[j].prevout.hash = txPrev->GetHash(); + } + BOOST_CHECK(SignSignature(keystore, *txPrev, tx, 0, SIGHASH_ALL)); + // Re-use same signature for other inputs + // (they don't have to be valid for this test) + for (unsigned int j = 1; j < tx.vin.size(); j++) + tx.vin[j].scriptSig = tx.vin[0].scriptSig; + + BOOST_CHECK(!orphanage.AddTx(MakeTransactionRef(tx), i)); + } + + // Test EraseOrphansFor: + for (NodeId i = 0; i < 3; i++) + { + size_t sizeBefore = orphanage.CountOrphans(); + orphanage.EraseForPeer(i); + BOOST_CHECK(orphanage.CountOrphans() < sizeBefore); + } + + // Test LimitOrphanTxSize() function: + orphanage.LimitOrphans(40); + BOOST_CHECK(orphanage.CountOrphans() <= 40); + orphanage.LimitOrphans(10); + BOOST_CHECK(orphanage.CountOrphans() <= 10); + orphanage.LimitOrphans(0); + BOOST_CHECK(orphanage.CountOrphans() == 0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index a9d661438c..3cbbec92d6 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -2,7 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <arith_uint256.h> #include <consensus/merkle.h> #include <merkleblock.h> #include <serialize.h> @@ -107,13 +106,13 @@ BOOST_AUTO_TEST_CASE(pmt_test1) BOOST_AUTO_TEST_CASE(pmt_malleability) { - std::vector<uint256> vTxid = { - ArithToUint256(1), ArithToUint256(2), - ArithToUint256(3), ArithToUint256(4), - ArithToUint256(5), ArithToUint256(6), - ArithToUint256(7), ArithToUint256(8), - ArithToUint256(9), ArithToUint256(10), - ArithToUint256(9), ArithToUint256(10), + std::vector<uint256> vTxid{ + uint256{1}, uint256{2}, + uint256{3}, uint256{4}, + uint256{5}, uint256{6}, + uint256{7}, uint256{8}, + uint256{9}, uint256{10}, + uint256{9}, uint256{10}, }; std::vector<bool> vMatch = {false, false, false, false, false, false, false, false, false, true, true, false}; diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 4fb7f9c405..df67841b66 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -31,8 +31,6 @@ #include <map> #include <string> -#include <boost/algorithm/string/classification.hpp> -#include <boost/algorithm/string/split.hpp> #include <boost/test/unit_test.hpp> #include <univalue.h> @@ -70,8 +68,7 @@ unsigned int ParseScriptFlags(std::string strFlags) { if (strFlags.empty() || strFlags == "NONE") return 0; unsigned int flags = 0; - std::vector<std::string> words; - boost::algorithm::split(words, strFlags, boost::algorithm::is_any_of(",")); + std::vector<std::string> words = SplitString(strFlags, ','); for (const std::string& word : words) { diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 13f17ca277..2fc71c2a6e 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -180,8 +180,8 @@ ChainTestingSetup::~ChainTestingSetup() m_node.connman.reset(); m_node.banman.reset(); m_node.addrman.reset(); + m_node.netgroupman.reset(); m_node.args = nullptr; - WITH_LOCK(::cs_main, UnloadBlockIndex(m_node.mempool.get(), *m_node.chainman)); m_node.mempool.reset(); m_node.scheduler.reset(); m_node.chainman.reset(); @@ -223,11 +223,12 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } - m_node.addrman = std::make_unique<AddrMan>(/*asmap=*/std::vector<bool>(), + m_node.netgroupman = std::make_unique<NetGroupManager>(/*asmap=*/std::vector<bool>()); + m_node.addrman = std::make_unique<AddrMan>(*m_node.netgroupman, /*deterministic=*/false, m_node.args->GetIntArg("-checkaddrman", 0)); m_node.banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); - m_node.connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); // Deterministic randomness for tests. + m_node.connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); // Deterministic randomness for tests. m_node.peerman = PeerManager::make(chainparams, *m_node.connman, *m_node.addrman, m_node.banman.get(), *m_node.chainman, *m_node.mempool, false); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index b5d8411e1d..1ca20fd848 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -226,17 +226,17 @@ BOOST_AUTO_TEST_CASE(util_Join) BOOST_AUTO_TEST_CASE(util_TrimString) { BOOST_CHECK_EQUAL(TrimString(" foo bar "), "foo bar"); - BOOST_CHECK_EQUAL(TrimString("\t \n \n \f\n\r\t\v\tfoo \n \f\n\r\t\v\tbar\t \n \f\n\r\t\v\t\n "), "foo \n \f\n\r\t\v\tbar"); + BOOST_CHECK_EQUAL(TrimStringView("\t \n \n \f\n\r\t\v\tfoo \n \f\n\r\t\v\tbar\t \n \f\n\r\t\v\t\n "), "foo \n \f\n\r\t\v\tbar"); BOOST_CHECK_EQUAL(TrimString("\t \n foo \n\tbar\t \n "), "foo \n\tbar"); - BOOST_CHECK_EQUAL(TrimString("\t \n foo \n\tbar\t \n ", "fobar"), "\t \n foo \n\tbar\t \n "); + BOOST_CHECK_EQUAL(TrimStringView("\t \n foo \n\tbar\t \n ", "fobar"), "\t \n foo \n\tbar\t \n "); BOOST_CHECK_EQUAL(TrimString("foo bar"), "foo bar"); - BOOST_CHECK_EQUAL(TrimString("foo bar", "fobar"), " "); + BOOST_CHECK_EQUAL(TrimStringView("foo bar", "fobar"), " "); BOOST_CHECK_EQUAL(TrimString(std::string("\0 foo \0 ", 8)), std::string("\0 foo \0", 7)); - BOOST_CHECK_EQUAL(TrimString(std::string(" foo ", 5)), std::string("foo", 3)); + BOOST_CHECK_EQUAL(TrimStringView(std::string(" foo ", 5)), std::string("foo", 3)); BOOST_CHECK_EQUAL(TrimString(std::string("\t\t\0\0\n\n", 6)), std::string("\0\0", 2)); - BOOST_CHECK_EQUAL(TrimString(std::string("\x05\x04\x03\x02\x01\x00", 6)), std::string("\x05\x04\x03\x02\x01\x00", 6)); + BOOST_CHECK_EQUAL(TrimStringView(std::string("\x05\x04\x03\x02\x01\x00", 6)), std::string("\x05\x04\x03\x02\x01\x00", 6)); BOOST_CHECK_EQUAL(TrimString(std::string("\x05\x04\x03\x02\x01\x00", 6), std::string("\x05\x04\x03\x02\x01", 5)), std::string("\0", 1)); - BOOST_CHECK_EQUAL(TrimString(std::string("\x05\x04\x03\x02\x01\x00", 6), std::string("\x05\x04\x03\x02\x01\x00", 6)), ""); + BOOST_CHECK_EQUAL(TrimStringView(std::string("\x05\x04\x03\x02\x01\x00", 6), std::string("\x05\x04\x03\x02\x01\x00", 6)), ""); } BOOST_AUTO_TEST_CASE(util_FormatParseISO8601DateTime) @@ -2349,6 +2349,55 @@ BOOST_AUTO_TEST_CASE(test_spanparsing) BOOST_CHECK_EQUAL(SpanToStr(results[3]), ""); } +BOOST_AUTO_TEST_CASE(test_SplitString) +{ + // Empty string. + { + std::vector<std::string> result = SplitString("", '-'); + BOOST_CHECK_EQUAL(result.size(), 1); + BOOST_CHECK_EQUAL(result[0], ""); + } + + // Empty items. + { + std::vector<std::string> result = SplitString("-", '-'); + BOOST_CHECK_EQUAL(result.size(), 2); + BOOST_CHECK_EQUAL(result[0], ""); + BOOST_CHECK_EQUAL(result[1], ""); + } + + // More empty items. + { + std::vector<std::string> result = SplitString("--", '-'); + BOOST_CHECK_EQUAL(result.size(), 3); + BOOST_CHECK_EQUAL(result[0], ""); + BOOST_CHECK_EQUAL(result[1], ""); + BOOST_CHECK_EQUAL(result[2], ""); + } + + // Separator is not present. + { + std::vector<std::string> result = SplitString("abc", '-'); + BOOST_CHECK_EQUAL(result.size(), 1); + BOOST_CHECK_EQUAL(result[0], "abc"); + } + + // Basic behavior. + { + std::vector<std::string> result = SplitString("a-b", '-'); + BOOST_CHECK_EQUAL(result.size(), 2); + BOOST_CHECK_EQUAL(result[0], "a"); + BOOST_CHECK_EQUAL(result[1], "b"); + } + + // Case-sensitivity of the separator. + { + std::vector<std::string> result = SplitString("AAA", 'a'); + BOOST_CHECK_EQUAL(result.size(), 1); + BOOST_CHECK_EQUAL(result[0], "AAA"); + } +} + BOOST_AUTO_TEST_CASE(test_LogEscapeMessage) { // ASCII and UTF-8 must pass through unaltered. @@ -2569,13 +2618,13 @@ BOOST_AUTO_TEST_CASE(message_hash) BOOST_AUTO_TEST_CASE(remove_prefix) { BOOST_CHECK_EQUAL(RemovePrefix("./util/system.h", "./"), "util/system.h"); - BOOST_CHECK_EQUAL(RemovePrefix("foo", "foo"), ""); + BOOST_CHECK_EQUAL(RemovePrefixView("foo", "foo"), ""); BOOST_CHECK_EQUAL(RemovePrefix("foo", "fo"), "o"); - BOOST_CHECK_EQUAL(RemovePrefix("foo", "f"), "oo"); + BOOST_CHECK_EQUAL(RemovePrefixView("foo", "f"), "oo"); BOOST_CHECK_EQUAL(RemovePrefix("foo", ""), "foo"); - BOOST_CHECK_EQUAL(RemovePrefix("fo", "foo"), "fo"); + BOOST_CHECK_EQUAL(RemovePrefixView("fo", "foo"), "fo"); BOOST_CHECK_EQUAL(RemovePrefix("f", "foo"), "f"); - BOOST_CHECK_EQUAL(RemovePrefix("", "foo"), ""); + BOOST_CHECK_EQUAL(RemovePrefixView("", "foo"), ""); BOOST_CHECK_EQUAL(RemovePrefix("", ""), ""); } diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index b0d7389d39..2a3990bb7c 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -93,7 +93,7 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); // Ensure our active chain is the snapshot chainstate. - BOOST_CHECK(chainman.IsSnapshotActive()); + BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.IsSnapshotActive())); curr_tip = ::g_best_block; diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 5d0ec593e3..6dc522b421 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -45,7 +45,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); BOOST_CHECK(!manager.IsSnapshotActive()); - BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(WITH_LOCK(::cs_main, return !manager.IsSnapshotValidated())); auto all = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); @@ -78,7 +78,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_CHECK(c2.ActivateBestChain(_, nullptr)); BOOST_CHECK(manager.IsSnapshotActive()); - BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(WITH_LOCK(::cs_main, return !manager.IsSnapshotValidated())); BOOST_CHECK_EQUAL(&c2, &manager.ActiveChainstate()); BOOST_CHECK(&c1 != &manager.ActiveChainstate()); auto all2 = manager.GetAll(); diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index a15094e5c8..e2a7cb4066 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -24,9 +24,7 @@ #include <set> #include <vector> -#include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/replace.hpp> -#include <boost/algorithm/string/split.hpp> #include <event2/buffer.h> #include <event2/bufferevent.h> @@ -308,7 +306,7 @@ std::map<std::string,std::string> ParseTorReplyMapping(const std::string &s) TorController::TorController(struct event_base* _base, const std::string& tor_control_center, const CService& target): base(_base), - m_tor_control_center(tor_control_center), conn(base), reconnect(true), reconnect_ev(0), + m_tor_control_center(tor_control_center), conn(base), reconnect(true), reconnect_ev(nullptr), reconnect_timeout(RECONNECT_TIMEOUT_START), m_target(target) { @@ -347,8 +345,8 @@ void TorController::get_socks_cb(TorControlConnection& _conn, const TorControlRe for (const auto& line : reply.lines) { if (0 == line.compare(0, 20, "net/listeners/socks=")) { const std::string port_list_str = line.substr(20); - std::vector<std::string> port_list; - boost::split(port_list, port_list_str, boost::is_any_of(" ")); + std::vector<std::string> port_list = SplitString(port_list_str, ' '); + for (auto& portstr : port_list) { if (portstr.empty()) continue; if ((portstr[0] == '"' || portstr[0] == '\'') && portstr.size() >= 2 && (*portstr.rbegin() == portstr[0])) { @@ -542,8 +540,10 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro if (l.first == "AUTH") { std::map<std::string,std::string> m = ParseTorReplyMapping(l.second); std::map<std::string,std::string>::iterator i; - if ((i = m.find("METHODS")) != m.end()) - boost::split(methods, i->second, boost::is_any_of(",")); + if ((i = m.find("METHODS")) != m.end()) { + std::vector<std::string> m_vec = SplitString(i->second, ','); + methods = std::set<std::string>(m_vec.begin(), m_vec.end()); + } if ((i = m.find("COOKIEFILE")) != m.end()) cookiefile = i->second; } else if (l.first == "VERSION") { @@ -582,7 +582,7 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro // _conn.Command("AUTHENTICATE " + HexStr(status_cookie.second), std::bind(&TorController::auth_cb, this, std::placeholders::_1, std::placeholders::_2)); cookie = std::vector<uint8_t>(status_cookie.second.begin(), status_cookie.second.end()); clientNonce = std::vector<uint8_t>(TOR_NONCE_SIZE, 0); - GetRandBytes(clientNonce.data(), TOR_NONCE_SIZE); + GetRandBytes(clientNonce); _conn.Command("AUTHCHALLENGE SAFECOOKIE " + HexStr(clientNonce), std::bind(&TorController::authchallenge_cb, this, std::placeholders::_1, std::placeholders::_2)); } else { if (status_cookie.first) { diff --git a/src/util/bytevectorhash.cpp b/src/util/bytevectorhash.cpp index f87d0e04b3..bc060a44c9 100644 --- a/src/util/bytevectorhash.cpp +++ b/src/util/bytevectorhash.cpp @@ -8,8 +8,8 @@ ByteVectorHash::ByteVectorHash() { - GetRandBytes(reinterpret_cast<unsigned char*>(&m_k0), sizeof(m_k0)); - GetRandBytes(reinterpret_cast<unsigned char*>(&m_k1), sizeof(m_k1)); + GetRandBytes({reinterpret_cast<unsigned char*>(&m_k0), sizeof(m_k0)}); + GetRandBytes({reinterpret_cast<unsigned char*>(&m_k1), sizeof(m_k1)}); } size_t ByteVectorHash::operator()(const std::vector<unsigned char>& input) const diff --git a/src/util/check.h b/src/util/check.h index 4ee65c8d34..aca957925a 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -18,8 +18,23 @@ class NonFatalCheckError : public std::runtime_error using std::runtime_error::runtime_error; }; +#define format_internal_error(msg, file, line, func, report) \ + strprintf("Internal bug detected: \"%s\"\n%s:%d (%s)\nPlease report this issue here: %s\n", \ + msg, file, line, func, report) + +/** Helper for CHECK_NONFATAL() */ +template <typename T> +T&& inline_check_non_fatal(T&& val, const char* file, int line, const char* func, const char* assertion) +{ + if (!(val)) { + throw NonFatalCheckError( + format_internal_error(assertion, file, line, func, PACKAGE_BUGREPORT)); + } + return std::forward<T>(val); +} + /** - * Throw a NonFatalCheckError when the condition evaluates to false + * Identity function. Throw a NonFatalCheckError when the condition evaluates to false * * This should only be used * - where the condition is assumed to be true, not for error handling or validating user input @@ -29,18 +44,8 @@ class NonFatalCheckError : public std::runtime_error * asserts or recoverable logic errors. A NonFatalCheckError in RPC code is caught and passed as a string to the RPC * caller, which can then report the issue to the developers. */ -#define CHECK_NONFATAL(condition) \ - do { \ - if (!(condition)) { \ - throw NonFatalCheckError( \ - strprintf("Internal bug detected: '%s'\n" \ - "%s:%d (%s)\n" \ - "You may report this issue here: %s\n", \ - (#condition), \ - __FILE__, __LINE__, __func__, \ - PACKAGE_BUGREPORT)); \ - } \ - } while (false) +#define CHECK_NONFATAL(condition) \ + inline_check_non_fatal(condition, __FILE__, __LINE__, __func__, #condition) #if defined(NDEBUG) #error "Cannot compile without assertions!" @@ -80,4 +85,13 @@ T&& inline_assertion_check(T&& val, [[maybe_unused]] const char* file, [[maybe_u */ #define Assume(val) inline_assertion_check<false>(val, __FILE__, __LINE__, __func__, #val) +/** + * NONFATAL_UNREACHABLE() is a macro that is used to mark unreachable code. It throws a NonFatalCheckError. + * This is used to mark code that is not yet implemented or is not yet reachable. + */ +#define NONFATAL_UNREACHABLE() \ + throw NonFatalCheckError( \ + format_internal_error("Unreachable code reached (non-fatal)", \ + __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT)) + #endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/message.cpp b/src/util/message.cpp index 2c7f0406f0..f58876f915 100644 --- a/src/util/message.cpp +++ b/src/util/message.cpp @@ -35,14 +35,13 @@ MessageVerificationResult MessageVerify( return MessageVerificationResult::ERR_ADDRESS_NO_KEY; } - bool invalid = false; - std::vector<unsigned char> signature_bytes = DecodeBase64(signature.c_str(), &invalid); - if (invalid) { + auto signature_bytes = DecodeBase64(signature); + if (!signature_bytes) { return MessageVerificationResult::ERR_MALFORMED_SIGNATURE; } CPubKey pubkey; - if (!pubkey.RecoverCompact(MessageHash(message), signature_bytes)) { + if (!pubkey.RecoverCompact(MessageHash(message), *signature_bytes)) { return MessageVerificationResult::ERR_PUBKEY_NOT_RECOVERED; } diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index 2cd7a426f8..8c4bc6e6f4 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -40,7 +40,7 @@ std::string FormatMoney(const CAmount n) std::optional<CAmount> ParseMoney(const std::string& money_string) { - if (!ValidAsCString(money_string)) { + if (!ContainsNoNUL(money_string)) { return std::nullopt; } const std::string str = TrimString(money_string); diff --git a/src/util/spanparsing.cpp b/src/util/spanparsing.cpp index 50f6aee419..8614bd1176 100644 --- a/src/util/spanparsing.cpp +++ b/src/util/spanparsing.cpp @@ -48,20 +48,4 @@ Span<const char> Expr(Span<const char>& sp) return ret; } -std::vector<Span<const char>> Split(const Span<const char>& sp, char sep) -{ - std::vector<Span<const char>> ret; - auto it = sp.begin(); - auto start = it; - while (it != sp.end()) { - if (*it == sep) { - ret.emplace_back(start, it); - start = it + 1; - } - ++it; - } - ret.emplace_back(start, it); - return ret; -} - } // namespace spanparsing diff --git a/src/util/spanparsing.h b/src/util/spanparsing.h index fa2e698e6d..ebec8714a7 100644 --- a/src/util/spanparsing.h +++ b/src/util/spanparsing.h @@ -43,7 +43,22 @@ Span<const char> Expr(Span<const char>& sp); * Note that this function does not care about braces, so splitting * "foo(bar(1),2),3) on ',' will return {"foo(bar(1)", "2)", "3)"}. */ -std::vector<Span<const char>> Split(const Span<const char>& sp, char sep); +template <typename T = Span<const char>> +std::vector<T> Split(const Span<const char>& sp, char sep) +{ + std::vector<T> ret; + auto it = sp.begin(); + auto start = it; + while (it != sp.end()) { + if (*it == sep) { + ret.emplace_back(start, it); + start = it + 1; + } + ++it; + } + ret.emplace_back(start, it); + return ret; +} } // namespace spanparsing diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 940fa90da2..35f62f0422 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -24,15 +24,15 @@ static const std::string SAFE_CHARS[] = CHARS_ALPHA_NUM + "!*'();:@&=+$,/?#[]-_.~%", // SAFE_CHARS_URI }; -std::string SanitizeString(const std::string& str, int rule) +std::string SanitizeString(std::string_view str, int rule) { - std::string strResult; - for (std::string::size_type i = 0; i < str.size(); i++) - { - if (SAFE_CHARS[rule].find(str[i]) != std::string::npos) - strResult.push_back(str[i]); + std::string result; + for (char c : str) { + if (SAFE_CHARS[rule].find(c) != std::string::npos) { + result.push_back(c); + } } - return strResult; + return result; } const signed char p_util_hexdigit[256] = @@ -58,56 +58,43 @@ signed char HexDigit(char c) return p_util_hexdigit[(unsigned char)c]; } -bool IsHex(const std::string& str) +bool IsHex(std::string_view str) { - for(std::string::const_iterator it(str.begin()); it != str.end(); ++it) - { - if (HexDigit(*it) < 0) - return false; + for (char c : str) { + if (HexDigit(c) < 0) return false; } return (str.size() > 0) && (str.size()%2 == 0); } -bool IsHexNumber(const std::string& str) +bool IsHexNumber(std::string_view str) { - size_t starting_location = 0; - if (str.size() > 2 && *str.begin() == '0' && *(str.begin()+1) == 'x') { - starting_location = 2; - } - for (const char c : str.substr(starting_location)) { + if (str.substr(0, 2) == "0x") str.remove_prefix(2); + for (char c : str) { if (HexDigit(c) < 0) return false; } // Return false for empty string or "0x". - return (str.size() > starting_location); + return str.size() > 0; } -std::vector<unsigned char> ParseHex(const char* psz) +std::vector<unsigned char> ParseHex(std::string_view str) { // convert hex dump to vector std::vector<unsigned char> vch; - while (true) - { - while (IsSpace(*psz)) - psz++; - signed char c = HexDigit(*psz++); - if (c == (signed char)-1) - break; - auto n{uint8_t(c << 4)}; - c = HexDigit(*psz++); - if (c == (signed char)-1) - break; - n |= c; - vch.push_back(n); + auto it = str.begin(); + while (it != str.end() && it + 1 != str.end()) { + if (IsSpace(*it)) { + ++it; + continue; + } + auto c1 = HexDigit(*(it++)); + auto c2 = HexDigit(*(it++)); + if (c1 < 0 || c2 < 0) break; + vch.push_back(uint8_t(c1 << 4) | c2); } return vch; } -std::vector<unsigned char> ParseHex(const std::string& str) -{ - return ParseHex(str.c_str()); -} - -void SplitHostPort(std::string in, uint16_t& portOut, std::string& hostOut) +void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) { size_t colon = in.find_last_of(':'); // if a : is found, and it either follows a [...], or no other : is in the string, treat it as port separator @@ -139,7 +126,7 @@ std::string EncodeBase64(Span<const unsigned char> input) return str; } -std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) +std::optional<std::vector<unsigned char>> DecodeBase64(std::string_view str) { static const int8_t decode64_table[256]{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -157,46 +144,23 @@ std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; - const char* e = p; - std::vector<uint8_t> val; - val.reserve(strlen(p)); - while (*p != 0) { - int x = decode64_table[(unsigned char)*p]; - if (x == -1) break; - val.push_back(uint8_t(x)); - ++p; - } + if (str.size() % 4 != 0) return {}; + /* One or two = characters at the end are permitted. */ + if (str.size() >= 1 && str.back() == '=') str.remove_suffix(1); + if (str.size() >= 1 && str.back() == '=') str.remove_suffix(1); std::vector<unsigned char> ret; - ret.reserve((val.size() * 3) / 4); - bool valid = ConvertBits<6, 8, false>([&](unsigned char c) { ret.push_back(c); }, val.begin(), val.end()); - - const char* q = p; - while (valid && *p != 0) { - if (*p != '=') { - valid = false; - break; - } - ++p; - } - valid = valid && (p - e) % 4 == 0 && p - q < 4; - if (pf_invalid) *pf_invalid = !valid; + ret.reserve((str.size() * 3) / 4); + bool valid = ConvertBits<6, 8, false>( + [&](unsigned char c) { ret.push_back(c); }, + str.begin(), str.end(), + [](char c) { return decode64_table[uint8_t(c)]; } + ); + if (!valid) return {}; return ret; } -std::string DecodeBase64(const std::string& str, bool* pf_invalid) -{ - if (!ValidAsCString(str)) { - if (pf_invalid) { - *pf_invalid = true; - } - return {}; - } - std::vector<unsigned char> vchRet = DecodeBase64(str.c_str(), pf_invalid); - return std::string((const char*)vchRet.data(), vchRet.size()); -} - std::string EncodeBase32(Span<const unsigned char> input, bool pad) { static const char *pbase32 = "abcdefghijklmnopqrstuvwxyz234567"; @@ -212,12 +176,12 @@ std::string EncodeBase32(Span<const unsigned char> input, bool pad) return str; } -std::string EncodeBase32(const std::string& str, bool pad) +std::string EncodeBase32(std::string_view str, bool pad) { return EncodeBase32(MakeUCharSpan(str), pad); } -std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid) +std::optional<std::vector<unsigned char>> DecodeBase32(std::string_view str) { static const int8_t decode32_table[256]{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -235,49 +199,29 @@ std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid) -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; - const char* e = p; - std::vector<uint8_t> val; - val.reserve(strlen(p)); - while (*p != 0) { - int x = decode32_table[(unsigned char)*p]; - if (x == -1) break; - val.push_back(uint8_t(x)); - ++p; - } + if (str.size() % 8 != 0) return {}; + /* 1, 3, 4, or 6 padding '=' suffix characters are permitted. */ + if (str.size() >= 1 && str.back() == '=') str.remove_suffix(1); + if (str.size() >= 2 && str.substr(str.size() - 2) == "==") str.remove_suffix(2); + if (str.size() >= 1 && str.back() == '=') str.remove_suffix(1); + if (str.size() >= 2 && str.substr(str.size() - 2) == "==") str.remove_suffix(2); std::vector<unsigned char> ret; - ret.reserve((val.size() * 5) / 8); - bool valid = ConvertBits<5, 8, false>([&](unsigned char c) { ret.push_back(c); }, val.begin(), val.end()); - - const char* q = p; - while (valid && *p != 0) { - if (*p != '=') { - valid = false; - break; - } - ++p; - } - valid = valid && (p - e) % 8 == 0 && p - q < 8; - if (pf_invalid) *pf_invalid = !valid; + ret.reserve((str.size() * 5) / 8); + bool valid = ConvertBits<5, 8, false>( + [&](unsigned char c) { ret.push_back(c); }, + str.begin(), str.end(), + [](char c) { return decode32_table[uint8_t(c)]; } + ); - return ret; -} + if (!valid) return {}; -std::string DecodeBase32(const std::string& str, bool* pf_invalid) -{ - if (!ValidAsCString(str)) { - if (pf_invalid) { - *pf_invalid = true; - } - return {}; - } - std::vector<unsigned char> vchRet = DecodeBase32(str.c_str(), pf_invalid); - return std::string((const char*)vchRet.data(), vchRet.size()); + return ret; } namespace { template <typename T> -bool ParseIntegral(const std::string& str, T* out) +bool ParseIntegral(std::string_view str, T* out) { static_assert(std::is_integral<T>::value); // Replicate the exact behavior of strtol/strtoll/strtoul/strtoull when @@ -296,37 +240,37 @@ bool ParseIntegral(const std::string& str, T* out) } }; // namespace -bool ParseInt32(const std::string& str, int32_t* out) +bool ParseInt32(std::string_view str, int32_t* out) { return ParseIntegral<int32_t>(str, out); } -bool ParseInt64(const std::string& str, int64_t* out) +bool ParseInt64(std::string_view str, int64_t* out) { return ParseIntegral<int64_t>(str, out); } -bool ParseUInt8(const std::string& str, uint8_t* out) +bool ParseUInt8(std::string_view str, uint8_t* out) { return ParseIntegral<uint8_t>(str, out); } -bool ParseUInt16(const std::string& str, uint16_t* out) +bool ParseUInt16(std::string_view str, uint16_t* out) { return ParseIntegral<uint16_t>(str, out); } -bool ParseUInt32(const std::string& str, uint32_t* out) +bool ParseUInt32(std::string_view str, uint32_t* out) { return ParseIntegral<uint32_t>(str, out); } -bool ParseUInt64(const std::string& str, uint64_t* out) +bool ParseUInt64(std::string_view str, uint64_t* out) { return ParseIntegral<uint64_t>(str, out); } -std::string FormatParagraph(const std::string& in, size_t width, size_t indent) +std::string FormatParagraph(std::string_view in, size_t width, size_t indent) { assert(width >= indent); std::stringstream out; @@ -395,7 +339,7 @@ static inline bool ProcessMantissaDigit(char ch, int64_t &mantissa, int &mantiss return true; } -bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out) +bool ParseFixedPoint(std::string_view val, int decimals, int64_t *amount_out) { int64_t mantissa = 0; int64_t exponent = 0; @@ -487,14 +431,14 @@ bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out) return true; } -std::string ToLower(const std::string& str) +std::string ToLower(std::string_view str) { std::string r; for (auto ch : str) r += ToLower(ch); return r; } -std::string ToUpper(const std::string& str) +std::string ToUpper(std::string_view str) { std::string r; for (auto ch : str) r += ToUpper(ch); @@ -522,7 +466,7 @@ std::string HexStr(const Span<const uint8_t> s) return rv; } -std::optional<uint64_t> ParseByteUnits(const std::string& str, ByteUnit default_multiplier) +std::optional<uint64_t> ParseByteUnits(std::string_view str, ByteUnit default_multiplier) { if (str.empty()) { return std::nullopt; diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 1f83fa3ffa..ebb6d88952 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -54,24 +54,21 @@ enum class ByteUnit : uint64_t { * @param[in] rule The set of safe chars to choose (default: least restrictive) * @return A new string without unsafe chars */ -std::string SanitizeString(const std::string& str, int rule = SAFE_CHARS_DEFAULT); -std::vector<unsigned char> ParseHex(const char* psz); -std::vector<unsigned char> ParseHex(const std::string& str); +std::string SanitizeString(std::string_view str, int rule = SAFE_CHARS_DEFAULT); +std::vector<unsigned char> ParseHex(std::string_view str); signed char HexDigit(char c); /* Returns true if each character in str is a hex character, and has an even * number of hex digits.*/ -bool IsHex(const std::string& str); +bool IsHex(std::string_view str); /** * Return true if the string is a hex number, optionally prefixed with "0x" */ -bool IsHexNumber(const std::string& str); -std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid = nullptr); -std::string DecodeBase64(const std::string& str, bool* pf_invalid = nullptr); +bool IsHexNumber(std::string_view str); +std::optional<std::vector<unsigned char>> DecodeBase64(std::string_view str); std::string EncodeBase64(Span<const unsigned char> input); inline std::string EncodeBase64(Span<const std::byte> input) { return EncodeBase64(MakeUCharSpan(input)); } -inline std::string EncodeBase64(const std::string& str) { return EncodeBase64(MakeUCharSpan(str)); } -std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid = nullptr); -std::string DecodeBase32(const std::string& str, bool* pf_invalid = nullptr); +inline std::string EncodeBase64(std::string_view str) { return EncodeBase64(MakeUCharSpan(str)); } +std::optional<std::vector<unsigned char>> DecodeBase32(std::string_view str); /** * Base32 encode. @@ -85,9 +82,9 @@ std::string EncodeBase32(Span<const unsigned char> input, bool pad = true); * If `pad` is true, then the output will be padded with '=' so that its length * is a multiple of 8. */ -std::string EncodeBase32(const std::string& str, bool pad = true); +std::string EncodeBase32(std::string_view str, bool pad = true); -void SplitHostPort(std::string in, uint16_t& portOut, std::string& hostOut); +void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut); // LocaleIndependentAtoi is provided for backwards compatibility reasons. // @@ -101,12 +98,12 @@ void SplitHostPort(std::string in, uint16_t& portOut, std::string& hostOut); // undefined behavior, while this function returns the maximum or minimum // values, respectively. template <typename T> -T LocaleIndependentAtoi(const std::string& str) +T LocaleIndependentAtoi(std::string_view str) { static_assert(std::is_integral<T>::value); T result; // Emulate atoi(...) handling of white space and leading +/-. - std::string s = TrimString(str); + std::string_view s = TrimStringView(str); if (!s.empty() && s[0] == '+') { if (s.length() >= 2 && s[1] == '-') { return 0; @@ -162,7 +159,7 @@ constexpr inline bool IsSpace(char c) noexcept { * parsed value is not in the range representable by the type T. */ template <typename T> -std::optional<T> ToIntegral(const std::string& str) +std::optional<T> ToIntegral(std::string_view str) { static_assert(std::is_integral<T>::value); T result; @@ -178,42 +175,42 @@ std::optional<T> ToIntegral(const std::string& str) * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -[[nodiscard]] bool ParseInt32(const std::string& str, int32_t *out); +[[nodiscard]] bool ParseInt32(std::string_view str, int32_t *out); /** * Convert string to signed 64-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -[[nodiscard]] bool ParseInt64(const std::string& str, int64_t *out); +[[nodiscard]] bool ParseInt64(std::string_view str, int64_t *out); /** * Convert decimal string to unsigned 8-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -[[nodiscard]] bool ParseUInt8(const std::string& str, uint8_t *out); +[[nodiscard]] bool ParseUInt8(std::string_view str, uint8_t *out); /** * Convert decimal string to unsigned 16-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if the entire string could not be parsed or if overflow or underflow occurred. */ -[[nodiscard]] bool ParseUInt16(const std::string& str, uint16_t* out); +[[nodiscard]] bool ParseUInt16(std::string_view str, uint16_t* out); /** * Convert decimal string to unsigned 32-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -[[nodiscard]] bool ParseUInt32(const std::string& str, uint32_t *out); +[[nodiscard]] bool ParseUInt32(std::string_view str, uint32_t *out); /** * Convert decimal string to unsigned 64-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. */ -[[nodiscard]] bool ParseUInt64(const std::string& str, uint64_t *out); +[[nodiscard]] bool ParseUInt64(std::string_view str, uint64_t *out); /** * Convert a span of bytes to a lower-case hexadecimal string. @@ -226,7 +223,7 @@ inline std::string HexStr(const Span<const std::byte> s) { return HexStr(MakeUCh * Format a paragraph of text to a fixed width, adding spaces for * indentation to any added line. */ -std::string FormatParagraph(const std::string& in, size_t width = 79, size_t indent = 0); +std::string FormatParagraph(std::string_view in, size_t width = 79, size_t indent = 0); /** * Timing-attack-resistant comparison. @@ -248,17 +245,28 @@ bool TimingResistantEqual(const T& a, const T& b) * @returns true on success, false on error. * @note The result must be in the range (-10^18,10^18), otherwise an overflow error will trigger. */ -[[nodiscard]] bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out); +[[nodiscard]] bool ParseFixedPoint(std::string_view, int decimals, int64_t *amount_out); + +namespace { +/** Helper class for the default infn argument to ConvertBits (just returns the input). */ +struct IntIdentity +{ + [[maybe_unused]] int operator()(int x) const { return x; } +}; + +} // namespace /** Convert from one power-of-2 number base to another. */ -template<int frombits, int tobits, bool pad, typename O, typename I> -bool ConvertBits(const O& outfn, I it, I end) { +template<int frombits, int tobits, bool pad, typename O, typename It, typename I = IntIdentity> +bool ConvertBits(O outfn, It it, It end, I infn = {}) { size_t acc = 0; size_t bits = 0; constexpr size_t maxv = (1 << tobits) - 1; constexpr size_t max_acc = (1 << (frombits + tobits - 1)) - 1; while (it != end) { - acc = ((acc << frombits) | *it) & max_acc; + int v = infn(*it); + if (v < 0) return false; + acc = ((acc << frombits) | v) & max_acc; bits += frombits; while (bits >= tobits) { bits -= tobits; @@ -298,7 +306,7 @@ constexpr char ToLower(char c) * @param[in] str the string to convert to lowercase. * @returns lowercased equivalent of str */ -std::string ToLower(const std::string& str); +std::string ToLower(std::string_view str); /** * Converts the given character to its uppercase equivalent. @@ -324,7 +332,7 @@ constexpr char ToUpper(char c) * @param[in] str the string to convert to uppercase. * @returns UPPERCASED EQUIVALENT OF str */ -std::string ToUpper(const std::string& str); +std::string ToUpper(std::string_view str); /** * Capitalizes the first character of the given string. @@ -348,6 +356,6 @@ std::string Capitalize(std::string str); * @returns optional uint64_t bytes from str or nullopt * if ToIntegral is false, str is empty, trailing whitespace or overflow */ -std::optional<uint64_t> ParseByteUnits(const std::string& str, ByteUnit default_multiplier); +std::optional<uint64_t> ParseByteUnits(std::string_view str, ByteUnit default_multiplier); #endif // BITCOIN_UTIL_STRENCODINGS_H diff --git a/src/util/string.h b/src/util/string.h index a3b8df8d78..36b9787db4 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -6,6 +6,7 @@ #define BITCOIN_UTIL_STRING_H #include <attributes.h> +#include <util/spanparsing.h> #include <algorithm> #include <array> @@ -15,17 +16,27 @@ #include <string> #include <vector> -[[nodiscard]] inline std::string TrimString(const std::string& str, const std::string& pattern = " \f\n\r\t\v") +[[nodiscard]] inline std::vector<std::string> SplitString(std::string_view str, char sep) +{ + return spanparsing::Split<std::string>(str, sep); +} + +[[nodiscard]] inline std::string_view TrimStringView(std::string_view str, std::string_view pattern = " \f\n\r\t\v") { std::string::size_type front = str.find_first_not_of(pattern); if (front == std::string::npos) { - return std::string(); + return {}; } std::string::size_type end = str.find_last_not_of(pattern); return str.substr(front, end - front + 1); } -[[nodiscard]] inline std::string RemovePrefix(const std::string& str, const std::string& prefix) +[[nodiscard]] inline std::string TrimString(std::string_view str, std::string_view pattern = " \f\n\r\t\v") +{ + return std::string(TrimStringView(str, pattern)); +} + +[[nodiscard]] inline std::string_view RemovePrefixView(std::string_view str, std::string_view prefix) { if (str.substr(0, prefix.size()) == prefix) { return str.substr(prefix.size()); @@ -33,6 +44,11 @@ return str; } +[[nodiscard]] inline std::string RemovePrefix(std::string_view str, std::string_view prefix) +{ + return std::string(RemovePrefixView(str, prefix)); +} + /** * Join a list of items * @@ -52,14 +68,14 @@ auto Join(const std::vector<T>& list, const BaseType& separator, UnaryOp unary_o return ret; } -template <typename T> -T Join(const std::vector<T>& list, const T& separator) +template <typename T, typename T2> +T Join(const std::vector<T>& list, const T2& separator) { return Join(list, separator, [](const T& i) { return i; }); } // Explicit overload needed for c_str arguments, which would otherwise cause a substitution failure in the template above. -inline std::string Join(const std::vector<std::string>& list, const std::string& separator) +inline std::string Join(const std::vector<std::string>& list, std::string_view separator) { return Join<std::string>(list, separator); } @@ -75,9 +91,12 @@ inline std::string MakeUnorderedList(const std::vector<std::string>& items) /** * Check if a string does not contain any embedded NUL (\0) characters */ -[[nodiscard]] inline bool ValidAsCString(const std::string& str) noexcept +[[nodiscard]] inline bool ContainsNoNUL(std::string_view str) noexcept { - return str.size() == strlen(str.c_str()); + for (auto c : str) { + if (c == 0) return false; + } + return true; } /** diff --git a/src/util/system.cpp b/src/util/system.cpp index a7e66defcd..f9a9ad3e20 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -853,8 +853,8 @@ static bool GetConfigOptions(std::istream& stream, const std::string& filepath, error = strprintf("parse error on line %i: %s, options in configuration file must be specified without leading -", linenr, str); return false; } else if ((pos = str.find('=')) != std::string::npos) { - std::string name = prefix + TrimString(str.substr(0, pos), pattern); - std::string value = TrimString(str.substr(pos + 1), pattern); + std::string name = prefix + TrimString(std::string_view{str}.substr(0, pos), pattern); + std::string_view value = TrimStringView(std::string_view{str}.substr(pos + 1), pattern); if (used_hash && name.find("rpcpassword") != std::string::npos) { error = strprintf("parse error on line %i, using # in rpcpassword can be ambiguous and should be avoided", linenr); return false; diff --git a/src/validation.cpp b/src/validation.cpp index 58686632f9..208bcee008 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -19,7 +19,6 @@ #include <deploymentstatus.h> #include <flatfile.h> #include <hash.h> -#include <index/blockfilterindex.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> @@ -106,6 +105,12 @@ const std::vector<std::string> CHECKLEVEL_DOC { "level 4 tries to reconnect the blocks", "each level includes the checks of the previous levels", }; +/** The number of blocks to keep below the deepest prune lock. + * There is nothing special about this number. It is higher than what we + * expect to see in regular mainnet reorgs, but not so high that it would + * noticeably interfere with the pruning mechanism. + * */ +static constexpr int PRUNE_LOCK_BUFFER{10}; /** * Mutex to guard access to validation specific variables, such as reading @@ -1916,7 +1921,7 @@ public: } }; -static ThresholdConditionCache warningcache[VERSIONBITS_NUM_BITS] GUARDED_BY(cs_main); +static std::array<ThresholdConditionCache, VERSIONBITS_NUM_BITS> warningcache GUARDED_BY(cs_main); static unsigned int GetBlockScriptFlags(const CBlockIndex& block_index, const Consensus::Params& consensusparams) { @@ -2338,12 +2343,24 @@ bool CChainState::FlushStateToDisk( CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(m_blockman.cs_LastBlockFile); if (fPruneMode && (m_blockman.m_check_for_pruning || nManualPruneHeight > 0) && !fReindex) { - // make sure we don't prune above the blockfilterindexes bestblocks + // make sure we don't prune above any of the prune locks bestblocks // pruning is height-based - int last_prune = m_chain.Height(); // last height we can prune - ForEachBlockFilterIndex([&](BlockFilterIndex& index) { - last_prune = std::max(1, std::min(last_prune, index.GetSummary().best_block_height)); - }); + int last_prune{m_chain.Height()}; // last height we can prune + std::optional<std::string> limiting_lock; // prune lock that actually was the limiting factor, only used for logging + + for (const auto& prune_lock : m_blockman.m_prune_locks) { + if (prune_lock.second.height_first == std::numeric_limits<int>::max()) continue; + // Remove the buffer and one additional block here to get actual height that is outside of the buffer + const int lock_height{prune_lock.second.height_first - PRUNE_LOCK_BUFFER - 1}; + last_prune = std::max(1, std::min(last_prune, lock_height)); + if (last_prune == lock_height) { + limiting_lock = prune_lock.first; + } + } + + if (limiting_lock) { + LogPrint(BCLog::PRUNE, "%s limited pruning to height %d\n", limiting_lock.value(), last_prune); + } if (nManualPruneHeight > 0) { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune (manual)", BCLog::BENCH); @@ -2533,7 +2550,7 @@ void CChainState::UpdateTip(const CBlockIndex* pindexNew) const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(bit); - ThresholdState state = checker.GetStateFor(pindex, m_params.GetConsensus(), warningcache[bit]); + ThresholdState state = checker.GetStateFor(pindex, m_params.GetConsensus(), warningcache.at(bit)); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { @@ -2581,6 +2598,18 @@ bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTr assert(flushed); } LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI); + + { + // Prune locks that began at or after the tip should be moved backward so they get a chance to reorg + const int max_height_first{pindexDelete->nHeight - 1}; + for (auto& prune_lock : m_blockman.m_prune_locks) { + if (prune_lock.second.height_first <= max_height_first) continue; + + prune_lock.second.height_first = max_height_first; + LogPrint(BCLog::PRUNE, "%s prune lock moved back to %d\n", prune_lock.first, max_height_first); + } + } + // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FlushStateMode::IF_NEEDED)) { return false; @@ -4114,20 +4143,6 @@ void CChainState::UnloadBlockIndex() setBlockIndexCandidates.clear(); } -// May NOT be used after any connections are up as much -// of the peer-processing logic assumes a consistent -// block index state -void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman) -{ - AssertLockHeld(::cs_main); - chainman.Unload(); - if (mempool) mempool->clear(); - g_versionbitscache.Clear(); - for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { - warningcache[b].clear(); - } -} - bool ChainstateManager::LoadBlockIndex() { AssertLockHeld(cs_main); @@ -5158,20 +5173,6 @@ bool ChainstateManager::IsSnapshotActive() const return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); } -void ChainstateManager::Unload() -{ - AssertLockHeld(::cs_main); - for (CChainState* chainstate : this->GetAll()) { - chainstate->m_chain.SetTip(nullptr); - chainstate->UnloadBlockIndex(); - } - - m_failed_blocks.clear(); - m_blockman.Unload(); - m_best_header = nullptr; - m_best_invalid = nullptr; -} - void ChainstateManager::MaybeRebalanceCaches() { AssertLockHeld(::cs_main); @@ -5202,3 +5203,15 @@ void ChainstateManager::MaybeRebalanceCaches() } } } + +ChainstateManager::~ChainstateManager() +{ + LOCK(::cs_main); + + // TODO: The version bits cache and warning cache should probably become + // non-globals + g_versionbitscache.Clear(); + for (auto& i : warningcache) { + i.clear(); + } +} diff --git a/src/validation.h b/src/validation.h index 2e7ab42f88..e3ea8617e7 100644 --- a/src/validation.h +++ b/src/validation.h @@ -134,8 +134,6 @@ extern arith_uint256 nMinimumChainWork; /** Documentation for argument 'checklevel'. */ extern const std::vector<std::string> CHECKLEVEL_DOC; -/** Unload database information */ -void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Run instances of script checking worker threads */ void StartScriptCheckWorkerThreads(int threads_num); /** Stop all of the script checking worker threads */ @@ -831,10 +829,9 @@ private: //! If true, the assumed-valid chainstate has been fully validated //! by the background validation chainstate. - bool m_snapshot_validated{false}; + bool m_snapshot_validated GUARDED_BY(::cs_main){false}; - CBlockIndex* m_best_invalid; - friend bool node::BlockManager::LoadBlockIndex(const Consensus::Params&); + CBlockIndex* m_best_invalid GUARDED_BY(::cs_main){nullptr}; //! Internal helper for ActivateSnapshot(). [[nodiscard]] bool PopulateAndValidateSnapshot( @@ -941,7 +938,7 @@ public: std::optional<uint256> SnapshotBlockhash() const; //! Is there a snapshot in use and has it been fully validated? - bool IsSnapshotValidated() const { return m_snapshot_validated; } + bool IsSnapshotValidated() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { return m_snapshot_validated; } /** * Process an incoming block. This only returns after the best known valid @@ -989,17 +986,11 @@ public: //! Load the block tree and coins database from disk, initializing state if we're running with -reindex bool LoadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - //! Unload block index and chain data before shutdown. - void Unload() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - //! Check to see if caches are out of balance and if so, call //! ResizeCoinsCaches() as needed. void MaybeRebalanceCaches() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - ~ChainstateManager() { - LOCK(::cs_main); - UnloadBlockIndex(/*mempool=*/nullptr, *this); - } + ~ChainstateManager(); }; using FopenFn = std::function<FILE*(const fs::path&, const char*)>; diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 0d0456af4b..7f709ffa3e 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -683,10 +683,10 @@ bool BerkeleyBatch::ReadAtCursor(CDataStream& ssKey, CDataStream& ssValue, bool& // Convert to streams ssKey.SetType(SER_DISK); ssKey.clear(); - ssKey.write({BytePtr(datKey.get_data()), datKey.get_size()}); + ssKey.write({AsBytePtr(datKey.get_data()), datKey.get_size()}); ssValue.SetType(SER_DISK); ssValue.clear(); - ssValue.write({BytePtr(datValue.get_data()), datValue.get_size()}); + ssValue.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); return true; } @@ -758,7 +758,7 @@ bool BerkeleyBatch::ReadKey(CDataStream&& key, CDataStream& value) SafeDbt datValue; int ret = pdb->get(activeTxn, datKey, datValue, 0); if (ret == 0 && datValue.get_data() != nullptr) { - value.write({BytePtr(datValue.get_data()), datValue.get_size()}); + value.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); return true; } return false; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 433759e086..0b236e2e48 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -64,7 +64,7 @@ static const size_t TOTAL_TRIES = 100000; std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& selection_target, const CAmount& cost_of_change) { - SelectionResult result(selection_target); + SelectionResult result(selection_target, SelectionAlgorithm::BNB); CAmount curr_value = 0; std::vector<size_t> curr_selection; // selected utxo indexes @@ -167,7 +167,7 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo std::optional<SelectionResult> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value, FastRandomContext& rng) { - SelectionResult result(target_value); + SelectionResult result(target_value, SelectionAlgorithm::SRD); std::vector<size_t> indexes; indexes.resize(utxo_pool.size()); @@ -249,7 +249,7 @@ static void ApproximateBestSubset(FastRandomContext& insecure_rand, const std::v std::optional<SelectionResult> KnapsackSolver(std::vector<OutputGroup>& groups, const CAmount& nTargetValue, CAmount change_target, FastRandomContext& rng) { - SelectionResult result(nTargetValue); + SelectionResult result(nTargetValue, SelectionAlgorithm::KNAPSACK); // List of values less than target std::optional<OutputGroup> lowest_larger; @@ -460,4 +460,17 @@ std::string COutput::ToString() const { return strprintf("COutput(%s, %d, %d) [%s]", outpoint.hash.ToString(), outpoint.n, depth, FormatMoney(txout.nValue)); } + +std::string GetAlgorithmName(const SelectionAlgorithm algo) +{ + switch (algo) + { + case SelectionAlgorithm::BNB: return "bnb"; + case SelectionAlgorithm::KNAPSACK: return "knapsack"; + case SelectionAlgorithm::SRD: return "srd"; + case SelectionAlgorithm::MANUAL: return "manual"; + // No default case to allow for compiler to warn + } + assert(false); +} } // namespace wallet diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index c1484c0a57..25c672eda0 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -239,21 +239,34 @@ struct OutputGroup */ [[nodiscard]] CAmount GenerateChangeTarget(CAmount payment_value, FastRandomContext& rng); +enum class SelectionAlgorithm : uint8_t +{ + BNB = 0, + KNAPSACK = 1, + SRD = 2, + MANUAL = 3, +}; + +std::string GetAlgorithmName(const SelectionAlgorithm algo); + struct SelectionResult { private: /** Set of inputs selected by the algorithm to use in the transaction */ std::set<COutput> m_selected_inputs; - /** The target the algorithm selected for. Note that this may not be equal to the recipient amount as it can include non-input fees */ - const CAmount m_target; /** Whether the input values for calculations should be the effective value (true) or normal value (false) */ bool m_use_effective{false}; /** The computed waste */ std::optional<CAmount> m_waste; public: - explicit SelectionResult(const CAmount target) - : m_target(target) {} + /** The target the algorithm selected for. Note that this may not be equal to the recipient amount as it can include non-input fees */ + const CAmount m_target; + /** The algorithm used to produce this result */ + const SelectionAlgorithm m_algo; + + explicit SelectionResult(const CAmount target, SelectionAlgorithm algo) + : m_target(target), m_algo(algo) {} SelectionResult() = delete; diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index b048ddfc6e..b4f01b00de 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -26,8 +26,6 @@ #include <tuple> #include <string> -#include <boost/algorithm/string.hpp> - #include <univalue.h> @@ -546,8 +544,7 @@ RPCHelpMan importwallet() if (line.empty() || line[0] == '#') continue; - std::vector<std::string> vstr; - boost::split(vstr, line, boost::is_any_of(" ")); + std::vector<std::string> vstr = SplitString(line, ' '); if (vstr.size() < 2) continue; CKey key = DecodeSecret(vstr[0]); @@ -915,7 +912,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d case TxoutType::WITNESS_V1_TAPROOT: return "unrecognized script"; } // no default case, so the compiler can warn about missing cases - CHECK_NONFATAL(false); + NONFATAL_UNREACHABLE(); } static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data, std::vector<CKeyID>& ordered_pubkeys) @@ -1748,13 +1745,13 @@ RPCHelpMan listdescriptors() {RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR, "desc", "Descriptor string representation"}, {RPCResult::Type::NUM, "timestamp", "The creation time of the descriptor"}, - {RPCResult::Type::BOOL, "active", "Activeness flag"}, - {RPCResult::Type::BOOL, "internal", true, "Whether this is an internal or external descriptor; defined only for active descriptors"}, - {RPCResult::Type::ARR_FIXED, "range", true, "Defined only for ranged descriptors", { + {RPCResult::Type::BOOL, "active", "Whether this descriptor is currently used to generate new addresses"}, + {RPCResult::Type::BOOL, "internal", /*optional=*/true, "True if this descriptor is used to generate change addresses. False if this descriptor is used to generate receiving addresses; defined only for active descriptors"}, + {RPCResult::Type::ARR_FIXED, "range", /*optional=*/true, "Defined only for ranged descriptors", { {RPCResult::Type::NUM, "", "Range start inclusive"}, {RPCResult::Type::NUM, "", "Range end inclusive"}, }}, - {RPCResult::Type::NUM, "next", true, "The next index to generate addresses from; defined only for ranged descriptors"}, + {RPCResult::Type::NUM, "next", /*optional=*/true, "The next index to generate addresses from; defined only for ranged descriptors"}, }}, }} }}, diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 1291663847..efb4d8fc7e 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -664,79 +664,75 @@ RPCHelpMan abortrescan(); Span<const CRPCCommand> GetWalletRPCCommands() { -// clang-format off -static const CRPCCommand commands[] = -{ // category actor (function) - // ------------------ ------------------------ - { "rawtransactions", &fundrawtransaction, }, - { "wallet", &abandontransaction, }, - { "wallet", &abortrescan, }, - { "wallet", &addmultisigaddress, }, - { "wallet", &backupwallet, }, - { "wallet", &bumpfee, }, - { "wallet", &psbtbumpfee, }, - { "wallet", &createwallet, }, - { "wallet", &restorewallet, }, - { "wallet", &dumpprivkey, }, - { "wallet", &dumpwallet, }, - { "wallet", &encryptwallet, }, - { "wallet", &getaddressesbylabel, }, - { "wallet", &getaddressinfo, }, - { "wallet", &getbalance, }, - { "wallet", &getnewaddress, }, - { "wallet", &getrawchangeaddress, }, - { "wallet", &getreceivedbyaddress, }, - { "wallet", &getreceivedbylabel, }, - { "wallet", &gettransaction, }, - { "wallet", &getunconfirmedbalance, }, - { "wallet", &getbalances, }, - { "wallet", &getwalletinfo, }, - { "wallet", &importaddress, }, - { "wallet", &importdescriptors, }, - { "wallet", &importmulti, }, - { "wallet", &importprivkey, }, - { "wallet", &importprunedfunds, }, - { "wallet", &importpubkey, }, - { "wallet", &importwallet, }, - { "wallet", &keypoolrefill, }, - { "wallet", &listaddressgroupings, }, - { "wallet", &listdescriptors, }, - { "wallet", &listlabels, }, - { "wallet", &listlockunspent, }, - { "wallet", &listreceivedbyaddress, }, - { "wallet", &listreceivedbylabel, }, - { "wallet", &listsinceblock, }, - { "wallet", &listtransactions, }, - { "wallet", &listunspent, }, - { "wallet", &listwalletdir, }, - { "wallet", &listwallets, }, - { "wallet", &loadwallet, }, - { "wallet", &lockunspent, }, - { "wallet", &newkeypool, }, - { "wallet", &removeprunedfunds, }, - { "wallet", &rescanblockchain, }, - { "wallet", &send, }, - { "wallet", &sendmany, }, - { "wallet", &sendtoaddress, }, - { "wallet", &sethdseed, }, - { "wallet", &setlabel, }, - { "wallet", &settxfee, }, - { "wallet", &setwalletflag, }, - { "wallet", &signmessage, }, - { "wallet", &signrawtransactionwithwallet, }, - { "wallet", &sendall, }, - { "wallet", &unloadwallet, }, - { "wallet", &upgradewallet, }, - { "wallet", &walletcreatefundedpsbt, }, + static const CRPCCommand commands[]{ + {"rawtransactions", &fundrawtransaction}, + {"wallet", &abandontransaction}, + {"wallet", &abortrescan}, + {"wallet", &addmultisigaddress}, + {"wallet", &backupwallet}, + {"wallet", &bumpfee}, + {"wallet", &psbtbumpfee}, + {"wallet", &createwallet}, + {"wallet", &restorewallet}, + {"wallet", &dumpprivkey}, + {"wallet", &dumpwallet}, + {"wallet", &encryptwallet}, + {"wallet", &getaddressesbylabel}, + {"wallet", &getaddressinfo}, + {"wallet", &getbalance}, + {"wallet", &getnewaddress}, + {"wallet", &getrawchangeaddress}, + {"wallet", &getreceivedbyaddress}, + {"wallet", &getreceivedbylabel}, + {"wallet", &gettransaction}, + {"wallet", &getunconfirmedbalance}, + {"wallet", &getbalances}, + {"wallet", &getwalletinfo}, + {"wallet", &importaddress}, + {"wallet", &importdescriptors}, + {"wallet", &importmulti}, + {"wallet", &importprivkey}, + {"wallet", &importprunedfunds}, + {"wallet", &importpubkey}, + {"wallet", &importwallet}, + {"wallet", &keypoolrefill}, + {"wallet", &listaddressgroupings}, + {"wallet", &listdescriptors}, + {"wallet", &listlabels}, + {"wallet", &listlockunspent}, + {"wallet", &listreceivedbyaddress}, + {"wallet", &listreceivedbylabel}, + {"wallet", &listsinceblock}, + {"wallet", &listtransactions}, + {"wallet", &listunspent}, + {"wallet", &listwalletdir}, + {"wallet", &listwallets}, + {"wallet", &loadwallet}, + {"wallet", &lockunspent}, + {"wallet", &newkeypool}, + {"wallet", &removeprunedfunds}, + {"wallet", &rescanblockchain}, + {"wallet", &send}, + {"wallet", &sendmany}, + {"wallet", &sendtoaddress}, + {"wallet", &sethdseed}, + {"wallet", &setlabel}, + {"wallet", &settxfee}, + {"wallet", &setwalletflag}, + {"wallet", &signmessage}, + {"wallet", &signrawtransactionwithwallet}, + {"wallet", &sendall}, + {"wallet", &unloadwallet}, + {"wallet", &upgradewallet}, + {"wallet", &walletcreatefundedpsbt}, #ifdef ENABLE_EXTERNAL_SIGNER - { "wallet", &walletdisplayaddress, }, + {"wallet", &walletdisplayaddress}, #endif // ENABLE_EXTERNAL_SIGNER - { "wallet", &walletlock, }, - { "wallet", &walletpassphrase, }, - { "wallet", &walletpassphrasechange, }, - { "wallet", &walletprocesspsbt, }, -}; -// clang-format on + {"wallet", &walletlock}, + {"wallet", &walletpassphrase}, + {"wallet", &walletpassphrasechange}, + {"wallet", &walletprocesspsbt}, + }; return commands; } } // namespace wallet diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 9e508f3a32..55c0a2cb7f 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -11,6 +11,7 @@ #include <util/fees.h> #include <util/moneystr.h> #include <util/rbf.h> +#include <util/trace.h> #include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/fees.h> @@ -435,9 +436,10 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec */ preset_inputs.Insert(out, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); } - SelectionResult result(nTargetValue); + SelectionResult result(nTargetValue, SelectionAlgorithm::MANUAL); result.AddInput(preset_inputs); if (result.GetSelectedValue() < nTargetValue) return std::nullopt; + result.ComputeAndSetWaste(coin_selection_params.m_cost_of_change); return result; } @@ -519,7 +521,7 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec // permissive CoinEligibilityFilter. std::optional<SelectionResult> res = [&] { // Pre-selected inputs already cover the target amount. - if (value_to_select <= 0) return std::make_optional(SelectionResult(nTargetValue)); + if (value_to_select <= 0) return std::make_optional(SelectionResult(nTargetValue, SelectionAlgorithm::MANUAL)); // If possible, fund the transaction with confirmed UTXOs only. Prefer at least six // confirmations on outputs received from other wallets and only spend confirmed change. @@ -573,6 +575,9 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec // Add preset inputs to result res->AddInput(preset_inputs); + if (res->m_algo == SelectionAlgorithm::MANUAL) { + res->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); + } return res; } @@ -788,6 +793,7 @@ static bool CreateTransactionInternal( error = _("Insufficient funds"); return false; } + TRACE5(coin_selection, selected_coins, wallet.GetName().c_str(), GetAlgorithmName(result->m_algo).c_str(), result->m_target, result->GetWaste(), result->GetSelectedValue()); // Always make a change output // We will reduce the fee from this change output later, and remove the output if it is too small. @@ -978,8 +984,10 @@ bool CreateTransaction( int nChangePosIn = nChangePosInOut; Assert(!tx); // tx is an out-param. TODO change the return type from bool to tx (or nullptr) bool res = CreateTransactionInternal(wallet, vecSend, tx, nFeeRet, nChangePosInOut, error, coin_control, fee_calc_out, sign); + TRACE4(coin_selection, normal_create_tx_internal, wallet.GetName().c_str(), res, nFeeRet, nChangePosInOut); // try with avoidpartialspends unless it's enabled already if (res && nFeeRet > 0 /* 0 means non-functional fee rate estimation */ && wallet.m_max_aps_fee > -1 && !coin_control.m_avoid_partial_spends) { + TRACE1(coin_selection, attempting_aps_create_tx, wallet.GetName().c_str()); CCoinControl tmp_cc = coin_control; tmp_cc.m_avoid_partial_spends = true; CAmount nFeeRet2; @@ -990,6 +998,7 @@ bool CreateTransaction( // if fee of this alternative one is within the range of the max fee, we use this one const bool use_aps = nFeeRet2 <= nFeeRet + wallet.m_max_aps_fee; wallet.WalletLogPrintf("Fee non-grouped = %lld, grouped = %lld, using %s\n", nFeeRet, nFeeRet2, use_aps ? "grouped" : "non-grouped"); + TRACE5(coin_selection, aps_create_tx_internal, wallet.GetName().c_str(), use_aps, res, nFeeRet2, nChangePosInOut2); if (use_aps) { tx = tx2; nFeeRet = nFeeRet2; diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 3f860289f9..2515df3177 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -405,7 +405,7 @@ bool SQLiteBatch::ReadKey(CDataStream&& key, CDataStream& value) return false; } // Leftmost column in result is index 0 - const std::byte* data{BytePtr(sqlite3_column_blob(m_read_stmt, 0))}; + const std::byte* data{AsBytePtr(sqlite3_column_blob(m_read_stmt, 0))}; size_t data_size(sqlite3_column_bytes(m_read_stmt, 0)); value.write({data, data_size}); @@ -497,10 +497,10 @@ bool SQLiteBatch::ReadAtCursor(CDataStream& key, CDataStream& value, bool& compl } // Leftmost column in result is index 0 - const std::byte* key_data{BytePtr(sqlite3_column_blob(m_cursor_stmt, 0))}; + const std::byte* key_data{AsBytePtr(sqlite3_column_blob(m_cursor_stmt, 0))}; size_t key_data_size(sqlite3_column_bytes(m_cursor_stmt, 0)); key.write({key_data, key_data_size}); - const std::byte* value_data{BytePtr(sqlite3_column_blob(m_cursor_stmt, 1))}; + const std::byte* value_data{AsBytePtr(sqlite3_column_blob(m_cursor_stmt, 1))}; size_t value_data_size(sqlite3_column_bytes(m_cursor_stmt, 1)); value.write({value_data, value_data_size}); return true; diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 2a08c8ab57..72e749477b 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -168,7 +168,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) FastRandomContext rand{}; // Setup std::vector<COutput> utxo_pool; - SelectionResult expected_result(CAmount(0)); + SelectionResult expected_result(CAmount(0), SelectionAlgorithm::BNB); ///////////////////////// // Known Outcome tests // diff --git a/src/wallet/test/wallet_crypto_tests.cpp b/src/wallet/test/wallet_crypto_tests.cpp index 166e27bab9..327c28412a 100644 --- a/src/wallet/test/wallet_crypto_tests.cpp +++ b/src/wallet/test/wallet_crypto_tests.cpp @@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE(passphrase) { std::string hash(GetRandHash().ToString()); std::vector<unsigned char> vchSalt(8); - GetRandBytes(vchSalt.data(), vchSalt.size()); + GetRandBytes(vchSalt); uint32_t rounds = InsecureRand32(); if (rounds > 30000) rounds = 30000; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index c3ae098aee..489599e2a0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -682,12 +682,12 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) CKeyingMaterial _vMasterKey; _vMasterKey.resize(WALLET_CRYPTO_KEY_SIZE); - GetStrongRandBytes(_vMasterKey.data(), WALLET_CRYPTO_KEY_SIZE); + GetStrongRandBytes(_vMasterKey); CMasterKey kMasterKey; kMasterKey.vchSalt.resize(WALLET_CRYPTO_SALT_SIZE); - GetStrongRandBytes(kMasterKey.vchSalt.data(), WALLET_CRYPTO_SALT_SIZE); + GetStrongRandBytes(kMasterKey.vchSalt); CCrypter crypter; int64_t nStartTime = GetTimeMillis(); @@ -2768,7 +2768,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri } else if (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS) { // Make it impossible to disable private keys after creation error = strprintf(_("Error loading %s: Private keys can only be disabled during creation"), walletFile); - return NULL; + return nullptr; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { if (spk_man->HavePrivateKeys()) { diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index f9f8b5a9dc..ec6d1cbba3 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -51,10 +51,8 @@ static RPCHelpMan getzmqnotifications() }; } -const CRPCCommand commands[] = -{ // category actor (function) - // ----------------- ----------------------- - { "zmq", &getzmqnotifications, }, +const CRPCCommand commands[]{ + {"zmq", &getzmqnotifications}, }; } // anonymous namespace diff --git a/test/README.md b/test/README.md index e5a184d23c..d69e515acf 100644 --- a/test/README.md +++ b/test/README.md @@ -327,7 +327,7 @@ test/lint/lint-files.py You can run all the shell-based lint tests by running: ``` -test/lint/lint-all.sh +test/lint/lint-all.py ``` # Writing functional tests diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py deleted file mode 100755 index c983ceda6f..0000000000 --- a/test/functional/feature_blockfilterindex_prune.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test blockfilterindex in conjunction with prune.""" -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - assert_greater_than, - assert_raises_rpc_error, -) - - -class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 1 - self.extra_args = [["-fastprune", "-prune=1", "-blockfilterindex=1"]] - - def sync_index(self, height): - expected = {'basic block filter index': {'synced': True, 'best_block_height': height}} - self.wait_until(lambda: self.nodes[0].getindexinfo() == expected) - - def run_test(self): - self.log.info("check if we can access a blockfilter when pruning is enabled but no blocks are actually pruned") - self.sync_index(height=200) - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) - self.generate(self.nodes[0], 500) - self.sync_index(height=700) - - self.log.info("prune some blocks") - pruneheight = self.nodes[0].pruneblockchain(400) - # the prune heights used here and below are magic numbers that are determined by the - # thresholds at which block files wrap, so they depend on disk serialization and default block file size. - assert_equal(pruneheight, 249) - - self.log.info("check if we can access the tips blockfilter when we have pruned some blocks") - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) - - self.log.info("check if we can access the blockfilter of a pruned block") - assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getblockhash(2))['filter']), 0) - - # mine and sync index up to a height that will later be the pruneheight - self.generate(self.nodes[0], 51) - self.sync_index(height=751) - - self.log.info("start node without blockfilterindex") - self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) - - self.log.info("make sure accessing the blockfilters throws an error") - assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[0].getblockfilter, self.nodes[0].getblockhash(2)) - self.generate(self.nodes[0], 749) - - self.log.info("prune exactly up to the blockfilterindexes best block while blockfilters are disabled") - pruneheight_2 = self.nodes[0].pruneblockchain(1000) - assert_equal(pruneheight_2, 751) - self.restart_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"]) - self.log.info("make sure that we can continue with the partially synced index after having pruned up to the index height") - self.sync_index(height=1500) - - self.log.info("prune below the blockfilterindexes best block while blockfilters are disabled") - self.restart_node(0, extra_args=["-fastprune", "-prune=1"]) - self.generate(self.nodes[0], 1000) - pruneheight_3 = self.nodes[0].pruneblockchain(2000) - assert_greater_than(pruneheight_3, pruneheight_2) - self.stop_node(0) - - self.log.info("make sure we get an init error when starting the node again with block filters") - self.nodes[0].assert_start_raises_init_error( - extra_args=["-fastprune", "-prune=1", "-blockfilterindex=1"], - expected_msg="Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", - ) - - self.log.info("make sure the node starts again with the -reindex arg") - self.start_node(0, extra_args=["-fastprune", "-prune=1", "-blockfilterindex", "-reindex"]) - - -if __name__ == '__main__': - FeatureBlockfilterindexPruneTest().main() diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index f865661894..251aa2114b 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -223,6 +223,20 @@ class CoinStatsIndexTest(BitcoinTestFramework): res10 = index_node.gettxoutsetinfo('muhash') assert(res8['txouts'] < res10['txouts']) + self.log.info("Test that the index works with -reindex") + + self.restart_node(1, extra_args=["-coinstatsindex", "-reindex"]) + res11 = index_node.gettxoutsetinfo('muhash') + assert_equal(res11, res10) + + self.log.info("Test that -reindex-chainstate is disallowed with coinstatsindex") + + self.nodes[1].assert_start_raises_init_error( + expected_msg='Error: -reindex-chainstate option is not compatible with -coinstatsindex. ' + 'Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', + extra_args=['-coinstatsindex', '-reindex-chainstate'], + ) + def _test_use_index_option(self): self.log.info("Test use_index option for nodes running the index") diff --git a/test/functional/feature_index_prune.py b/test/functional/feature_index_prune.py new file mode 100755 index 0000000000..2bf57db923 --- /dev/null +++ b/test/functional/feature_index_prune.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test indices in conjunction with prune.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, + p2p_port, +) + + +class FeatureIndexPruneTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 4 + self.extra_args = [ + ["-fastprune", "-prune=1", "-blockfilterindex=1"], + ["-fastprune", "-prune=1", "-coinstatsindex=1"], + ["-fastprune", "-prune=1", "-blockfilterindex=1", "-coinstatsindex=1"], + [] + ] + + def sync_index(self, height): + expected_filter = { + 'basic block filter index': {'synced': True, 'best_block_height': height}, + } + self.wait_until(lambda: self.nodes[0].getindexinfo() == expected_filter) + + expected_stats = { + 'coinstatsindex': {'synced': True, 'best_block_height': height} + } + self.wait_until(lambda: self.nodes[1].getindexinfo() == expected_stats) + + expected = {**expected_filter, **expected_stats} + self.wait_until(lambda: self.nodes[2].getindexinfo() == expected) + + def reconnect_nodes(self): + self.connect_nodes(0,1) + self.connect_nodes(0,2) + self.connect_nodes(0,3) + + def mine_batches(self, blocks): + n = blocks // 250 + for _ in range(n): + self.generate(self.nodes[0], 250) + self.generate(self.nodes[0], blocks % 250) + self.sync_blocks() + + def restart_without_indices(self): + for i in range(3): + self.restart_node(i, extra_args=["-fastprune", "-prune=1"]) + self.reconnect_nodes() + + def run_test(self): + filter_nodes = [self.nodes[0], self.nodes[2]] + stats_nodes = [self.nodes[1], self.nodes[2]] + + self.log.info("check if we can access blockfilters and coinstats when pruning is enabled but no blocks are actually pruned") + self.sync_index(height=200) + tip = self.nodes[0].getbestblockhash() + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(tip)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']) + + self.mine_batches(500) + self.sync_index(height=700) + + self.log.info("prune some blocks") + for node in self.nodes[:2]: + with node.assert_debug_log(['limited pruning to height 689']): + pruneheight_new = node.pruneblockchain(400) + # the prune heights used here and below are magic numbers that are determined by the + # thresholds at which block files wrap, so they depend on disk serialization and default block file size. + assert_equal(pruneheight_new, 249) + + self.log.info("check if we can access the tips blockfilter and coinstats when we have pruned some blocks") + tip = self.nodes[0].getbestblockhash() + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(tip)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=tip)['muhash']) + + self.log.info("check if we can access the blockfilter and coinstats of a pruned block") + height_hash = self.nodes[0].getblockhash(2) + for node in filter_nodes: + assert_greater_than(len(node.getblockfilter(height_hash)['filter']), 0) + for node in stats_nodes: + assert(node.gettxoutsetinfo(hash_type="muhash", hash_or_height=height_hash)['muhash']) + + # mine and sync index up to a height that will later be the pruneheight + self.generate(self.nodes[0], 51) + self.sync_index(height=751) + + self.restart_without_indices() + + self.log.info("make sure trying to access the indices throws errors") + for node in filter_nodes: + msg = "Index is not enabled for filtertype basic" + assert_raises_rpc_error(-1, msg, node.getblockfilter, height_hash) + for node in stats_nodes: + msg = "Querying specific block heights requires coinstatsindex" + assert_raises_rpc_error(-8, msg, node.gettxoutsetinfo, "muhash", height_hash) + + self.mine_batches(749) + + self.log.info("prune exactly up to the indices best blocks while the indices are disabled") + for i in range(3): + pruneheight_2 = self.nodes[i].pruneblockchain(1000) + assert_equal(pruneheight_2, 751) + # Restart the nodes again with the indices activated + self.restart_node(i, extra_args=self.extra_args[i]) + + self.log.info("make sure that we can continue with the partially synced indices after having pruned up to the index height") + self.sync_index(height=1500) + + self.log.info("prune further than the indices best blocks while the indices are disabled") + self.restart_without_indices() + self.mine_batches(1000) + + for i in range(3): + pruneheight_3 = self.nodes[i].pruneblockchain(2000) + assert_greater_than(pruneheight_3, pruneheight_2) + self.stop_node(i) + + self.log.info("make sure we get an init error when starting the nodes again with the indices") + filter_msg = "Error: basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" + stats_msg = "Error: coinstatsindex best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)" + for i, msg in enumerate([filter_msg, stats_msg, filter_msg]): + self.nodes[i].assert_start_raises_init_error(extra_args=self.extra_args[i], expected_msg=msg) + + self.log.info("make sure the nodes start again with the indices and an additional -reindex arg") + ip_port = "127.0.0.1:" + str(p2p_port(3)) + for i in range(3): + # The nodes need to be reconnected to the non-pruning node upon restart, otherwise they will be stuck + restart_args = self.extra_args[i]+["-reindex", f"-connect={ip_port}"] + self.restart_node(i, extra_args=restart_args) + + self.sync_blocks(timeout=300) + + for node in self.nodes[:2]: + with node.assert_debug_log(['limited pruning to height 2489']): + pruneheight_new = node.pruneblockchain(2500) + assert_equal(pruneheight_new, 2006) + + self.log.info("ensure that prune locks don't prevent indices from failing in a reorg scenario") + with self.nodes[0].assert_debug_log(['basic block filter index prune lock moved back to 2480']): + self.nodes[3].invalidateblock(self.nodes[0].getblockhash(2480)) + self.generate(self.nodes[3], 30) + self.sync_blocks() + + +if __name__ == '__main__': + FeatureIndexPruneTest().main() diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index bf19384279..4110526d15 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -138,10 +138,6 @@ class PruneTest(BitcoinTestFramework): extra_args=['-prune=550', '-txindex'], ) self.nodes[0].assert_start_raises_init_error( - expected_msg='Error: Prune mode is incompatible with -coinstatsindex.', - extra_args=['-prune=550', '-coinstatsindex'], - ) - self.nodes[0].assert_start_raises_init_error( expected_msg='Error: Prune mode is incompatible with -reindex-chainstate. Use full -reindex instead.', extra_args=['-prune=550', '-reindex-chainstate'], ) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 30c9e0c9cd..95dc40cb52 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -326,6 +326,10 @@ class RESTTest (BitcoinTestFramework): # Check that there are our submitted transactions in the TX memory pool json_obj = self.test_rest_request("/mempool/contents") + raw_mempool_verbose = self.nodes[0].getrawmempool(verbose=True) + + assert_equal(json_obj, raw_mempool_verbose) + for i, tx in enumerate(txs): assert tx in json_obj assert_equal(json_obj[tx]['spentby'], txs[i + 1:i + 2]) diff --git a/test/functional/interface_usdt_coinselection.py b/test/functional/interface_usdt_coinselection.py new file mode 100755 index 0000000000..ef32feda99 --- /dev/null +++ b/test/functional/interface_usdt_coinselection.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Tests the coin_selection:* tracepoint API interface. + See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-coin_selection +""" + +# Test will be skipped if we don't have bcc installed +try: + from bcc import BPF, USDT # type: ignore[import] +except ImportError: + pass +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_raises_rpc_error, +) + +coinselection_tracepoints_program = """ +#include <uapi/linux/ptrace.h> + +#define WALLET_NAME_LENGTH 16 +#define ALGO_NAME_LENGTH 16 + +struct event_data +{ + u8 type; + char wallet_name[WALLET_NAME_LENGTH]; + + // selected coins event + char algo[ALGO_NAME_LENGTH]; + s64 target; + s64 waste; + s64 selected_value; + + // create tx event + bool success; + s64 fee; + s32 change_pos; + + // aps create tx event + bool use_aps; +}; + +BPF_QUEUE(coin_selection_events, struct event_data, 1024); + +int trace_selected_coins(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 1; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg_p(2, ctx, &data.algo, ALGO_NAME_LENGTH); + bpf_usdt_readarg(3, ctx, &data.target); + bpf_usdt_readarg(4, ctx, &data.waste); + bpf_usdt_readarg(5, ctx, &data.selected_value); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_normal_create_tx(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 2; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg(2, ctx, &data.success); + bpf_usdt_readarg(3, ctx, &data.fee); + bpf_usdt_readarg(4, ctx, &data.change_pos); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_attempt_aps(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 3; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + coin_selection_events.push(&data, 0); + return 0; +} + +int trace_aps_create_tx(struct pt_regs *ctx) { + struct event_data data; + __builtin_memset(&data, 0, sizeof(data)); + data.type = 4; + bpf_usdt_readarg_p(1, ctx, &data.wallet_name, WALLET_NAME_LENGTH); + bpf_usdt_readarg(2, ctx, &data.use_aps); + bpf_usdt_readarg(3, ctx, &data.success); + bpf_usdt_readarg(4, ctx, &data.fee); + bpf_usdt_readarg(5, ctx, &data.change_pos); + coin_selection_events.push(&data, 0); + return 0; +} +""" + + +class CoinSelectionTracepointTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def skip_test_if_missing_module(self): + self.skip_if_platform_not_linux() + self.skip_if_no_bitcoind_tracepoints() + self.skip_if_no_python_bcc() + self.skip_if_no_bpf_permissions() + self.skip_if_no_wallet() + + def get_tracepoints(self, expected_types): + events = [] + try: + for i in range(0, len(expected_types) + 1): + event = self.bpf["coin_selection_events"].pop() + assert_equal(event.wallet_name.decode(), self.default_wallet_name) + assert_equal(event.type, expected_types[i]) + events.append(event) + else: + # If the loop exits successfully instead of throwing a KeyError, then we have had + # more events than expected. There should be no more than len(expected_types) events. + assert False + except KeyError: + assert_equal(len(events), len(expected_types)) + return events + + + def determine_selection_from_usdt(self, events): + success = None + use_aps = None + algo = None + waste = None + change_pos = None + + is_aps = False + sc_events = [] + for event in events: + if event.type == 1: + if not is_aps: + algo = event.algo.decode() + waste = event.waste + sc_events.append(event) + elif event.type == 2: + success = event.success + if not is_aps: + change_pos = event.change_pos + elif event.type == 3: + is_aps = True + elif event.type == 4: + assert is_aps + if event.use_aps: + use_aps = True + assert_equal(len(sc_events), 2) + algo = sc_events[1].algo.decode() + waste = sc_events[1].waste + change_pos = event.change_pos + return success, use_aps, algo, waste, change_pos + + def run_test(self): + self.log.info("hook into the coin_selection tracepoints") + ctx = USDT(pid=self.nodes[0].process.pid) + ctx.enable_probe(probe="coin_selection:selected_coins", fn_name="trace_selected_coins") + ctx.enable_probe(probe="coin_selection:normal_create_tx_internal", fn_name="trace_normal_create_tx") + ctx.enable_probe(probe="coin_selection:attempting_aps_create_tx", fn_name="trace_attempt_aps") + ctx.enable_probe(probe="coin_selection:aps_create_tx_internal", fn_name="trace_aps_create_tx") + self.bpf = BPF(text=coinselection_tracepoints_program, usdt_contexts=[ctx], debug=0) + + self.log.info("Prepare wallets") + self.generate(self.nodes[0], 101) + wallet = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + + self.log.info("Sending a transaction should result in all tracepoints") + # We should have 5 tracepoints in the order: + # 1. selected_coins (type 1) + # 2. normal_create_tx_internal (type 2) + # 3. attempting_aps_create_tx (type 3) + # 4. selected_coins (type 1) + # 5. aps_create_tx_internal (type 4) + wallet.sendtoaddress(wallet.getnewaddress(), 10) + events = self.get_tracepoints([1, 2, 3, 1, 4]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, True) + assert_greater_than(change_pos, -1) + + self.log.info("Failing to fund results in 1 tracepoint") + # We should have 1 tracepoints in the order + # 1. normal_create_tx_internal (type 2) + assert_raises_rpc_error(-6, "Insufficient funds", wallet.sendtoaddress, wallet.getnewaddress(), 102 * 50) + events = self.get_tracepoints([2]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, False) + + self.log.info("Explicitly enabling APS results in 2 tracepoints") + # We should have 2 tracepoints in the order + # 1. selected_coins (type 1) + # 2. normal_create_tx_internal (type 2) + wallet.setwalletflag("avoid_reuse") + wallet.sendtoaddress(address=wallet.getnewaddress(), amount=10, avoid_reuse=True) + events = self.get_tracepoints([1, 2]) + success, use_aps, algo, waste, change_pos = self.determine_selection_from_usdt(events) + assert_equal(success, True) + assert_equal(use_aps, None) + + self.bpf.cleanup() + + +if __name__ == '__main__': + CoinSelectionTracepointTest().main() diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 3218a9b14a..e2e9b6dcb2 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -21,8 +21,19 @@ from test_framework.p2p import ( P2P_SERVICES, ) from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_greater_than_or_equal +) + +ONE_MINUTE = 60 +TEN_MINUTES = 10 * ONE_MINUTE +ONE_HOUR = 60 * ONE_MINUTE +TWO_HOURS = 2 * ONE_HOUR +ONE_DAY = 24 * ONE_HOUR +ADDR_DESTINATIONS_THRESHOLD = 4 class AddrReceiver(P2PInterface): num_ipv4_received = 0 @@ -85,6 +96,9 @@ class AddrTest(BitcoinTestFramework): self.relay_tests() self.inbound_blackhole_tests() + self.destination_rotates_once_in_24_hours_test() + self.destination_rotates_more_than_once_over_several_days_test() + # This test populates the addrman, which can impact the node's behavior # in subsequent tests self.getaddr_tests() @@ -362,6 +376,56 @@ class AddrTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() + def get_nodes_that_received_addr(self, peer, receiver_peer, addr_receivers, + time_interval_1, time_interval_2): + + # Clean addr response related to the initial getaddr. There is no way to avoid initial + # getaddr because the peer won't self-announce then. + for addr_receiver in addr_receivers: + addr_receiver.num_ipv4_received = 0 + + for _ in range(10): + self.mocktime += time_interval_1 + self.msg.addrs[0].time = self.mocktime + TEN_MINUTES + self.nodes[0].setmocktime(self.mocktime) + with self.nodes[0].assert_debug_log(['received: addr (31 bytes) peer=0']): + peer.send_and_ping(self.msg) + self.mocktime += time_interval_2 + self.nodes[0].setmocktime(self.mocktime) + receiver_peer.sync_with_ping() + return [node for node in addr_receivers if node.addr_received()] + + def destination_rotates_once_in_24_hours_test(self): + self.restart_node(0, []) + + self.log.info('Test within 24 hours an addr relay destination is rotated at most once') + self.mocktime = int(time.time()) + self.msg = self.setup_addr_msg(1) + self.addr_receivers = [] + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + receiver_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + addr_receivers = [self.nodes[0].add_p2p_connection(AddrReceiver()) for _ in range(20)] + nodes_received_addr = self.get_nodes_that_received_addr(peer, receiver_peer, addr_receivers, 0, TWO_HOURS) # 10 intervals of 2 hours + # Per RelayAddress, we would announce these addrs to 2 destinations per day. + # Since it's at most one rotation, at most 4 nodes can receive ADDR. + assert_greater_than_or_equal(ADDR_DESTINATIONS_THRESHOLD, len(nodes_received_addr)) + self.nodes[0].disconnect_p2ps() + + def destination_rotates_more_than_once_over_several_days_test(self): + self.restart_node(0, []) + + self.log.info('Test after several days an addr relay destination is rotated more than once') + self.msg = self.setup_addr_msg(1) + peer = self.nodes[0].add_p2p_connection(P2PInterface()) + receiver_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + addr_receivers = [self.nodes[0].add_p2p_connection(AddrReceiver()) for _ in range(20)] + # 10 intervals of 1 day (+ 1 hour, which should be enough to cover 30-min Poisson in most cases) + nodes_received_addr = self.get_nodes_that_received_addr(peer, receiver_peer, addr_receivers, ONE_DAY, ONE_HOUR) + # Now that there should have been more than one rotation, more than + # ADDR_DESTINATIONS_THRESHOLD nodes should have received ADDR. + assert_greater_than(len(nodes_received_addr), ADDR_DESTINATIONS_THRESHOLD) + self.nodes[0].disconnect_p2ps() + if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index e45cef65bd..2ffc825acd 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -250,6 +250,12 @@ class CompactFiltersTest(BitcoinTestFramework): msg = "Error: Cannot set -peerblockfilters without -blockfilterindex." self.nodes[0].assert_start_raises_init_error(expected_msg=msg) + self.log.info("Test -blockfilterindex with -reindex-chainstate raises an error") + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: -reindex-chainstate option is not compatible with -blockfilterindex. ' + 'Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.', + extra_args=['-blockfilterindex', '-reindex-chainstate'], + ) def compute_last_header(prev_header, hashes): """Compute the last filter header from a starting header and a sequence of filter hashes.""" diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index f64aae7223..f6ee6a5215 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -27,7 +27,7 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test CHECK_NONFATAL") assert_raises_rpc_error( -1, - 'Internal bug detected: \'request.params[9].get_str() != "trigger_internal_bug"\'', + 'Internal bug detected: "request.params[9].get_str() != "trigger_internal_bug""', lambda: node.echo(arg9='trigger_internal_bug'), ) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index b037807b53..444e56610e 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -615,8 +619,8 @@ class PSBTTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") wallet = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = self.nodes[0].importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -634,7 +638,7 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-4, "Insufficient funds", wallet.walletcreatefundedpsbt, [ext_utxo], {self.nodes[0].getnewaddress(): 15}) # But funding should work when the solving data is provided - psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + psbt = wallet.walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {"add_inputs": True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}}) signed = wallet.walletprocesspsbt(psbt['psbt']) assert not signed['complete'] signed = self.nodes[0].walletprocesspsbt(signed['psbt']) @@ -655,10 +659,11 @@ class PSBTTest(BitcoinTestFramework): break psbt_in = dec["inputs"][input_idx] # Calculate the input weight - # (prevout + sequence + length of scriptSig + 2 bytes buffer) * 4 + len of scriptwitness + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptwitness = len(psbt_in["final_scriptwitness"]["hex"]) // 2 if "final_scriptwitness" in psbt_in else 0 - input_weight = ((41 + len_scriptsig + 2) * 4) + len_scriptwitness + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness low_input_weight = input_weight // 2 high_input_weight = input_weight * 2 diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 746b6ede20..bb6f7d7e1a 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -82,6 +82,7 @@ EXTENDED_SCRIPTS = [ # Longest test should go first, to favor running tests in parallel 'feature_pruning.py', 'feature_dbcrash.py', + 'feature_index_prune.py', ] BASE_SCRIPTS = [ @@ -169,6 +170,7 @@ BASE_SCRIPTS = [ 'wallet_reorgsrestore.py', 'interface_http.py', 'interface_rpc.py', + 'interface_usdt_coinselection.py', 'interface_usdt_net.py', 'interface_usdt_utxocache.py', 'interface_usdt_validation.py', @@ -331,7 +333,6 @@ BASE_SCRIPTS = [ 'feature_help.py', 'feature_shutdown.py', 'p2p_ibd_txrelay.py', - 'feature_blockfilterindex_prune.py' # Don't append tests at the end to avoid merge conflicts # Put them in a random line within the section that fits their approximate run-time ] diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index dcf2e98638..12480d4d1e 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -29,7 +29,7 @@ class CreateWalletTest(BitcoinTestFramework): self.log.info("Run createwallet with invalid parameters.") # Run createwallet with invalid parameters. This must not prevent a new wallet with the same name from being created with the correct parameters. assert_raises_rpc_error(-4, "Passphrase provided but private keys are disabled. A passphrase is only used to encrypt private keys, so cannot be used for wallets with private keys disabled.", - self.nodes[0].createwallet, wallet_name='w0', descriptors=True, disable_private_keys=True, passphrase="passphrase") + self.nodes[0].createwallet, wallet_name='w0', disable_private_keys=True, passphrase="passphrase") self.nodes[0].createwallet(wallet_name='w0') w0 = node.get_wallet_rpc('w0') diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 48b92796fc..a7f4f9ffaf 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -26,9 +26,6 @@ class ReceivedByTest(BitcoinTestFramework): self.skip_if_no_cli() def run_test(self): - # Generate block to get out of IBD - self.generate(self.nodes[0], 1) - # save the number of coinbase reward addresses so far num_cb_reward_addresses = len(self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True)) @@ -172,7 +169,7 @@ class ReceivedByTest(BitcoinTestFramework): address = self.nodes[0].getnewaddress(label) reward = Decimal("25") - self.generatetoaddress(self.nodes[0], 1, address, sync_fun=self.no_op) + self.generatetoaddress(self.nodes[0], 1, address) hash = self.nodes[0].getbestblockhash() self.log.info("getreceivedbyaddress returns nothing with defaults") @@ -212,7 +209,7 @@ class ReceivedByTest(BitcoinTestFramework): {"label": label, "amount": reward}) self.log.info("Generate 100 more blocks") - self.generate(self.nodes[0], COINBASE_MATURITY, sync_fun=self.no_op) + self.generate(self.nodes[0], COINBASE_MATURITY) self.log.info("getreceivedbyaddress returns reward with defaults") balance = self.nodes[0].getreceivedbyaddress(address) diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index 86e36be8f7..07baa0595e 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -10,6 +10,10 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create from test_framework.key import ECKey +from test_framework.messages import ( + ser_compact_size, + WITNESS_SCALE_FACTOR, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -488,8 +492,8 @@ class WalletSendTest(BitcoinTestFramework): self.nodes[1].createwallet("extfund") ext_fund = self.nodes[1].get_wallet_rpc("extfund") - # Make a weird but signable script. sh(pkh()) descriptor accomplishes this - desc = descsum_create("sh(pkh({}))".format(privkey)) + # Make a weird but signable script. sh(wsh(pkh())) descriptor accomplishes this + desc = descsum_create("sh(wsh(pkh({})))".format(privkey)) if self.options.descriptors: res = ext_fund.importdescriptors([{"desc": desc, "timestamp": "now"}]) else: @@ -507,7 +511,7 @@ class WalletSendTest(BitcoinTestFramework): self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, expect_error=(-4, "Insufficient funds")) # But funding should work when the solving data is provided - res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}) + res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"], addr_info["embedded"]["embedded"]["scriptPubKey"]]}) signed = ext_wallet.walletprocesspsbt(res["psbt"]) signed = ext_fund.walletprocesspsbt(res["psbt"]) assert signed["complete"] @@ -526,10 +530,11 @@ class WalletSendTest(BitcoinTestFramework): break psbt_in = dec["inputs"][input_idx] # Calculate the input weight - # (prevout + sequence + length of scriptSig + 2 bytes buffer) * 4 + len of scriptwitness + # (prevout + sequence + length of scriptSig + scriptsig + 1 byte buffer) * WITNESS_SCALE_FACTOR + num scriptWitness stack items + (length of stack item + stack item) * N stack items + 1 byte buffer len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptwitness = len(psbt_in["final_scriptwitness"]["hex"]) // 2 if "final_scriptwitness" in psbt_in else 0 - input_weight = ((41 + len_scriptsig + 2) * 4) + len_scriptwitness + len_scriptsig += len(ser_compact_size(len_scriptsig)) + 1 + len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(psbt_in["final_scriptwitness"]) + 1) if "final_scriptwitness" in psbt_in else 0 + input_weight = ((40 + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness # Input weight error conditions assert_raises_rpc_error( diff --git a/test/lint/README.md b/test/lint/README.md index f4165f908e..1f683c10b3 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -39,6 +39,6 @@ To do so, add the upstream repository as remote: git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git ``` -lint-all.sh +lint-all.py =========== Calls other scripts with the `lint-` prefix. diff --git a/test/lint/lint-all.py b/test/lint/lint-all.py new file mode 100755 index 0000000000..c280ba2db2 --- /dev/null +++ b/test/lint/lint-all.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# This script runs all test/lint/lint-* files, and fails if any exit +# with a non-zero status code. + +from glob import glob +from pathlib import Path +from subprocess import run + +exit_code = 0 +mod_path = Path(__file__).parent +for lint in glob(f"{mod_path}/lint-*"): + if lint != __file__: + result = run([lint]) + if result.returncode != 0: + print(f"^---- failure generated from {lint.split('/')[-1]}") + exit_code |= result.returncode + +exit(exit_code) diff --git a/test/lint/lint-all.sh b/test/lint/lint-all.sh deleted file mode 100755 index fa37fa51c6..0000000000 --- a/test/lint/lint-all.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/lint-* files, and fails if any exit -# with a non-zero status code. - -# This script is intentionally locale dependent by not setting "export LC_ALL=C" -# in order to allow for the executed lint scripts to opt in or opt out of locale -# dependence themselves. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -EXIT_CODE=0 - -for f in "${SCRIPTDIR}"/lint-*; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - EXIT_CODE=1 - fi - fi -done - -exit ${EXIT_CODE} diff --git a/test/lint/lint-assertions.py b/test/lint/lint-assertions.py new file mode 100755 index 0000000000..195ff33d11 --- /dev/null +++ b/test/lint/lint-assertions.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for assertions with obvious side effects. + +import sys +import subprocess + + +def git_grep(params: [], error_msg: ""): + try: + output = subprocess.check_output(["git", "grep", *params], universal_newlines=True, encoding="utf8") + print(error_msg) + print(output) + return 1 + except subprocess.CalledProcessError as ex1: + if ex1.returncode > 1: + raise ex1 + return 0 + + +def main(): + # PRE31-C (SEI CERT C Coding Standard): + # "Assertions should not contain assignments, increment, or decrement operators." + exit_code = git_grep([ + "-E", + r"[^_]assert\(.*(\+\+|\-\-|[^=!<>]=[^=!<>]).*\);", + "--", + "*.cpp", + "*.h", + ], "Assertions should not have side effects:") + + # Aborting the whole process is undesirable for RPC code. So nonfatal + # checks should be used over assert. See: src/util/check.h + # src/rpc/server.cpp is excluded from this check since it's mostly meta-code. + exit_code |= git_grep([ + "-nE", + r"\<(A|a)ss(ume|ert) *\(.*\);", + "--", + "src/rpc/", + "src/wallet/rpc*", + ":(exclude)src/rpc/server.cpp", + ], "CHECK_NONFATAL(condition) or NONFATAL_UNREACHABLE should be used instead of assert for RPC code.") + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-assertions.sh b/test/lint/lint-assertions.sh deleted file mode 100755 index 2860f5621b..0000000000 --- a/test/lint/lint-assertions.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for assertions with obvious side effects. - -export LC_ALL=C - -EXIT_CODE=0 - -# PRE31-C (SEI CERT C Coding Standard): -# "Assertions should not contain assignments, increment, or decrement operators." -OUTPUT=$(git grep -E '[^_]assert\(.*(\+\+|\-\-|[^=!<>]=[^=!<>]).*\);' -- "*.cpp" "*.h") -if [[ ${OUTPUT} != "" ]]; then - echo "Assertions should not have side effects:" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi - -# Macro CHECK_NONFATAL(condition) should be used instead of assert for RPC code, where it -# is undesirable to crash the whole program. See: src/util/check.h -# src/rpc/server.cpp is excluded from this check since it's mostly meta-code. -OUTPUT=$(git grep -nE '\<(A|a)ssert *\(.*\);' -- "src/rpc/" "src/wallet/rpc*" ":(exclude)src/rpc/server.cpp") -if [[ ${OUTPUT} != "" ]]; then - echo "CHECK_NONFATAL(condition) should be used instead of assert for RPC code." - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py new file mode 100755 index 0000000000..e04909c0a5 --- /dev/null +++ b/test/lint/lint-circular-dependencies.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for circular dependencies + +import glob +import os +import re +import subprocess +import sys + +EXPECTED_CIRCULAR_DEPENDENCIES = ( + "chainparamsbase -> util/system -> chainparamsbase", + "node/blockstorage -> validation -> node/blockstorage", + "index/coinstatsindex -> node/coinstats -> index/coinstatsindex", + "policy/fees -> txmempool -> policy/fees", + "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel", + "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel", + "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog", + "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel", + "wallet/fees -> wallet/wallet -> wallet/fees", + "wallet/wallet -> wallet/walletdb -> wallet/wallet", + "node/coinstats -> validation -> node/coinstats", +) + +CODE_DIR = "src" + + +def main(): + circular_dependencies = [] + exit_code = 0 + os.chdir( + CODE_DIR + ) # We change dir before globbing since glob.glob's root_dir option is only available in Python 3.10 + + # Using glob.glob since subprocess.run's globbing won't work without shell=True + files = [] + for path in ["*", "*/*", "*/*/*"]: + for extension in ["h", "cpp"]: + files.extend(glob.glob(f"{path}.{extension}")) + + command = ["python3", "../contrib/devtools/circular-dependencies.py", *files] + dependencies_output = subprocess.run( + command, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + + for dependency_str in dependencies_output.stdout.rstrip().split("\n"): + circular_dependencies.append( + re.sub("^Circular dependency: ", "", dependency_str) + ) + + # Check for an unexpected dependencies + for dependency in circular_dependencies: + if dependency not in EXPECTED_CIRCULAR_DEPENDENCIES: + exit_code = 1 + print( + f'A new circular dependency in the form of "{dependency}" appears to have been introduced.\n', + file=sys.stderr, + ) + + # Check for missing expected dependencies + for expected_dependency in EXPECTED_CIRCULAR_DEPENDENCIES: + if expected_dependency not in circular_dependencies: + exit_code = 1 + print( + f'Good job! The circular dependency "{expected_dependency}" is no longer present.', + ) + print( + f"Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in {__file__}", + ) + print( + "to make sure this circular dependency is not accidentally reintroduced.\n", + ) + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh deleted file mode 100755 index 69185090d1..0000000000 --- a/test/lint/lint-circular-dependencies.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for circular dependencies - -export LC_ALL=C - -EXPECTED_CIRCULAR_DEPENDENCIES=( - "chainparamsbase -> util/system -> chainparamsbase" - "node/blockstorage -> validation -> node/blockstorage" - "index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex" - "index/base -> validation -> index/blockfilterindex -> index/base" - "index/coinstatsindex -> node/coinstats -> index/coinstatsindex" - "policy/fees -> txmempool -> policy/fees" - "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" - "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel" - "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog" - "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel" - "wallet/fees -> wallet/wallet -> wallet/fees" - "wallet/wallet -> wallet/walletdb -> wallet/wallet" - "node/coinstats -> validation -> node/coinstats" -) - -EXIT_CODE=0 - -CIRCULAR_DEPENDENCIES=() - -IFS=$'\n' -for CIRC in $(cd src && ../contrib/devtools/circular-dependencies.py {*,*/*,*/*/*}.{h,cpp} | sed -e 's/^Circular dependency: //'); do - CIRCULAR_DEPENDENCIES+=( "$CIRC" ) - IS_EXPECTED_CIRC=0 - for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_EXPECTED_CIRC} == 0 ]]; then - echo "A new circular dependency in the form of \"${CIRC}\" appears to have been introduced." - echo - EXIT_CODE=1 - fi -done - -for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - IS_PRESENT_EXPECTED_CIRC=0 - for CIRC in "${CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_PRESENT_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_PRESENT_EXPECTED_CIRC} == 0 ]]; then - echo "Good job! The circular dependency \"${EXPECTED_CIRC}\" is no longer present." - echo "Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in $0" - echo "to make sure this circular dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -exit ${EXIT_CODE} diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py new file mode 100755 index 0000000000..5a36da11fd --- /dev/null +++ b/test/lint/lint-format-strings.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# + +""" +Lint format strings: This program checks that the number of arguments passed +to a variadic format string function matches the number of format specifiers +in the format string. +""" + +import subprocess +import re +import sys + +FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS = [ + 'FatalError,0', + 'fprintf,1', + 'tfm::format,1', # Assuming tfm::::format(std::ostream&, ... + 'LogConnectFailure,1', + 'LogPrint,1', + 'LogPrintf,0', + 'printf,0', + 'snprintf,2', + 'sprintf,1', + 'strprintf,0', + 'vfprintf,1', + 'vprintf,1', + 'vsnprintf,1', + 'vsprintf,1', + 'WalletLogPrintf,0', +] +RUN_LINT_FILE = 'test/lint/run-lint-format-strings.py' + +def check_doctest(): + command = [ + 'python3', + '-m', + 'doctest', + RUN_LINT_FILE, + ] + try: + subprocess.run(command, check = True) + except subprocess.CalledProcessError: + sys.exit(1) + +def get_matching_files(function_name): + command = [ + 'git', + 'grep', + '--full-name', + '-l', + function_name, + '--', + '*.c', + '*.cpp', + '*.h', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + check_doctest() + for s in FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS: + function_name, skip_arguments = s.split(',') + matching_files = get_matching_files(function_name) + + matching_files_filtered = [] + for matching_file in matching_files: + if not re.search('^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)', matching_file): + matching_files_filtered.append(matching_file) + matching_files_filtered.sort() + + run_lint_args = [ + RUN_LINT_FILE, + '--skip-arguments', + skip_arguments, + function_name, + ] + run_lint_args.extend(matching_files_filtered) + + try: + subprocess.run(run_lint_args, check = True) + except subprocess.CalledProcessError: + exit_code = 1 + + sys.exit(exit_code) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh deleted file mode 100755 index 73730f16b3..0000000000 --- a/test/lint/lint-format-strings.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Lint format strings: This program checks that the number of arguments passed -# to a variadic format string function matches the number of format specifiers -# in the format string. - -export LC_ALL=C - -FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS=( - "FatalError,0" - "fprintf,1" - "tfm::format,1" # Assuming tfm::::format(std::ostream&, ... - "LogConnectFailure,1" - "LogPrint,1" - "LogPrintf,0" - "printf,0" - "snprintf,2" - "sprintf,1" - "strprintf,0" - "vfprintf,1" - "vprintf,1" - "vsnprintf,1" - "vsprintf,1" - "WalletLogPrintf,0" -) - -EXIT_CODE=0 -if ! python3 -m doctest "test/lint/run-lint-format-strings.py"; then - EXIT_CODE=1 -fi -for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do - IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}" - for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do - MATCHING_FILES+=("${MATCHING_FILE}") - done - if ! "test/lint/run-lint-format-strings.py" --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py new file mode 100755 index 0000000000..86284517d5 --- /dev/null +++ b/test/lint/lint-include-guards.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check include guards. +""" + +import re +import sys +from subprocess import check_output +from typing import List + + +HEADER_ID_PREFIX = 'BITCOIN_' +HEADER_ID_SUFFIX = '_H' + +EXCLUDE_FILES_WITH_PREFIX = ['src/crypto/ctaes', + 'src/leveldb', + 'src/crc32c', + 'src/secp256k1', + 'src/minisketch', + 'src/univalue', + 'src/tinyformat.h', + 'src/bench/nanobench.h', + 'src/test/fuzz/FuzzedDataProvider.h'] + + +def _get_header_file_lst() -> List[str]: + """ Helper function to get a list of header filepaths to be + checked for include guards. + """ + git_cmd_lst = ['git', 'ls-files', '--', '*.h'] + header_file_lst = check_output( + git_cmd_lst).decode('utf-8').splitlines() + + header_file_lst = [hf for hf in header_file_lst + if not any(ef in hf for ef + in EXCLUDE_FILES_WITH_PREFIX)] + + return header_file_lst + + +def _get_header_id(header_file: str) -> str: + """ Helper function to get the header id from a header file + string. + + eg: 'src/wallet/walletdb.h' -> 'BITCOIN_WALLET_WALLETDB_H' + + Args: + header_file: Filepath to header file. + + Returns: + The header id. + """ + header_id_base = header_file.split('/')[1:] + header_id_base = '_'.join(header_id_base) + header_id_base = header_id_base.replace('.h', '').replace('-', '_') + header_id_base = header_id_base.upper() + + header_id = f'{HEADER_ID_PREFIX}{header_id_base}{HEADER_ID_SUFFIX}' + + return header_id + + +def main(): + exit_code = 0 + + header_file_lst = _get_header_file_lst() + for header_file in header_file_lst: + header_id = _get_header_id(header_file) + + regex_pattern = f'^#(ifndef|define|endif //) {header_id}' + + with open(header_file, 'r', encoding='utf-8') as f: + header_file_contents = f.readlines() + + count = 0 + for header_file_contents_string in header_file_contents: + include_guard_lst = re.findall( + regex_pattern, header_file_contents_string) + + count += len(include_guard_lst) + + if count != 3: + print(f'{header_file} seems to be missing the expected ' + 'include guard:') + print(f' #ifndef {header_id}') + print(f' #define {header_id}') + print(' ...') + print(f' #endif // {header_id}\n') + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh deleted file mode 100755 index f14218aa74..0000000000 --- a/test/lint/lint-include-guards.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check include guards. - -export LC_ALL=C -HEADER_ID_PREFIX="BITCOIN_" -HEADER_ID_SUFFIX="_H" - -REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/)" - -EXIT_CODE=0 -for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") -do - HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") - HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" - if [[ $(grep --count --extended-regexp "^#(ifndef|define|endif //) ${HEADER_ID}" "${HEADER_FILE}") != 3 ]]; then - echo "${HEADER_FILE} seems to be missing the expected include guard:" - echo " #ifndef ${HEADER_ID}" - echo " #define ${HEADER_ID}" - echo " ..." - echo " #endif // ${HEADER_ID}" - echo - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-locale-dependence.py b/test/lint/lint-locale-dependence.py new file mode 100755 index 0000000000..2abf1be6b3 --- /dev/null +++ b/test/lint/lint-locale-dependence.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt +# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, +# whereas no such call is made in bitcoind. +# +# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale +# specified by the user's LC_ALL (or LC_*) environment variable as the new +# C locale. +# +# In contrast, bitcoind does not opt in to localization -- no call to +# setlocale(LC_ALL, "") is made and the environment variables LC_* are +# thus ignored. +# +# This results in situations where bitcoind is guaranteed to be running +# with the classic locale ("C") whereas the locale of bitcoin-qt will vary +# depending on the user's environment variables. +# +# An example: Assuming the environment variable LC_ALL=de_DE then the +# call std::to_string(1.23) will return "1.230000" in bitcoind but +# "1,230000" in bitcoin-qt. +# +# From the Qt documentation: +# "On Unix/Linux Qt is configured to use the system locale settings by default. +# This can cause a conflict when using POSIX functions, for instance, when +# converting between data types such as floats and strings, since the notation +# may differ between locales. To get around this problem, call the POSIX function +# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication +# or QCoreApplication to reset the locale that is used for number formatting to +# "C"-locale." +# +# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and +# https://stackoverflow.com/a/34878283 for more details. +# +# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. + +import re +import sys + +from subprocess import check_output, CalledProcessError + + +KNOWN_VIOLATIONS = [ + "src/dbwrapper.cpp:.*vsnprintf", + "src/test/dbwrapper_tests.cpp:.*snprintf", + "src/test/fuzz/locale.cpp:.*setlocale", + "src/test/fuzz/string.cpp:.*strtol", + "src/test/fuzz/string.cpp:.*strtoul", + "src/test/util_tests.cpp:.*strtoll" +] + +REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS = [ + "src/crypto/ctaes/", + "src/leveldb/", + "src/secp256k1/", + "src/minisketch/", + "src/tinyformat.h", + "src/univalue/" +] + +LOCALE_DEPENDENT_FUNCTIONS = [ + "alphasort", # LC_COLLATE (via strcoll) + "asctime", # LC_TIME (directly) + "asprintf", # (via vasprintf) + "atof", # LC_NUMERIC (via strtod) + "atoi", # LC_NUMERIC (via strtol) + "atol", # LC_NUMERIC (via strtol) + "atoll", # (via strtoll) + "atoq", + "btowc", # LC_CTYPE (directly) + "ctime", # (via asctime or localtime) + "dprintf", # (via vdprintf) + "fgetwc", + "fgetws", + "fold_case", # boost::locale::fold_case + "fprintf", # (via vfprintf) + "fputwc", + "fputws", + "fscanf", # (via __vfscanf) + "fwprintf", # (via __vfwprintf) + "getdate", # via __getdate_r => isspace // __localtime_r + "getwc", + "getwchar", + "is_digit", # boost::algorithm::is_digit + "is_space", # boost::algorithm::is_space + "isalnum", # LC_CTYPE + "isalpha", # LC_CTYPE + "isblank", # LC_CTYPE + "iscntrl", # LC_CTYPE + "isctype", # LC_CTYPE + "isdigit", # LC_CTYPE + "isgraph", # LC_CTYPE + "islower", # LC_CTYPE + "isprint", # LC_CTYPE + "ispunct", # LC_CTYPE + "isspace", # LC_CTYPE + "isupper", # LC_CTYPE + "iswalnum", # LC_CTYPE + "iswalpha", # LC_CTYPE + "iswblank", # LC_CTYPE + "iswcntrl", # LC_CTYPE + "iswctype", # LC_CTYPE + "iswdigit", # LC_CTYPE + "iswgraph", # LC_CTYPE + "iswlower", # LC_CTYPE + "iswprint", # LC_CTYPE + "iswpunct", # LC_CTYPE + "iswspace", # LC_CTYPE + "iswupper", # LC_CTYPE + "iswxdigit", # LC_CTYPE + "isxdigit", # LC_CTYPE + "localeconv", # LC_NUMERIC + LC_MONETARY + "mblen", # LC_CTYPE + "mbrlen", + "mbrtowc", + "mbsinit", + "mbsnrtowcs", + "mbsrtowcs", + "mbstowcs", # LC_CTYPE + "mbtowc", # LC_CTYPE + "mktime", + "normalize", # boost::locale::normalize + "printf", # LC_NUMERIC + "putwc", + "putwchar", + "scanf", # LC_NUMERIC + "setlocale", + "snprintf", + "sprintf", + "sscanf", + "std::locale::global", + "std::to_string", + "stod", + "stof", + "stoi", + "stol", + "stold", + "stoll", + "stoul", + "stoull", + "strcasecmp", + "strcasestr", + "strcoll", # LC_COLLATE + #"strerror", + "strfmon", + "strftime", # LC_TIME + "strncasecmp", + "strptime", + "strtod", # LC_NUMERIC + "strtof", + "strtoimax", + "strtol", # LC_NUMERIC + "strtold", + "strtoll", + "strtoq", + "strtoul", # LC_NUMERIC + "strtoull", + "strtoumax", + "strtouq", + "strxfrm", # LC_COLLATE + "swprintf", + "to_lower", # boost::locale::to_lower + "to_title", # boost::locale::to_title + "to_upper", # boost::locale::to_upper + "tolower", # LC_CTYPE + "toupper", # LC_CTYPE + "towctrans", + "towlower", # LC_CTYPE + "towupper", # LC_CTYPE + "trim", # boost::algorithm::trim + "trim_left", # boost::algorithm::trim_left + "trim_right", # boost::algorithm::trim_right + "ungetwc", + "vasprintf", + "vdprintf", + "versionsort", + "vfprintf", + "vfscanf", + "vfwprintf", + "vprintf", + "vscanf", + "vsnprintf", + "vsprintf", + "vsscanf", + "vswprintf", + "vwprintf", + "wcrtomb", + "wcscasecmp", + "wcscoll", # LC_COLLATE + "wcsftime", # LC_TIME + "wcsncasecmp", + "wcsnrtombs", + "wcsrtombs", + "wcstod", # LC_NUMERIC + "wcstof", + "wcstoimax", + "wcstol", # LC_NUMERIC + "wcstold", + "wcstoll", + "wcstombs", # LC_CTYPE + "wcstoul", # LC_NUMERIC + "wcstoull", + "wcstoumax", + "wcswidth", + "wcsxfrm", # LC_COLLATE + "wctob", + "wctomb", # LC_CTYPE + "wctrans", + "wctype", + "wcwidth", + "wprintf" +] + + +def find_locale_dependent_function_uses(): + regexp_locale_dependent_functions = "|".join(LOCALE_DEPENDENT_FUNCTIONS) + exclude_args = [":(exclude)" + excl for excl in REGEXP_EXTERNAL_DEPENDENCIES_EXCLUSIONS] + git_grep_command = ["git", "grep", "-E", "[^a-zA-Z0-9_\\`'\"<>](" + regexp_locale_dependent_functions + "(_r|_s)?)[^a-zA-Z0-9_\\`'\"<>]", "--", "*.cpp", "*.h"] + exclude_args + git_grep_output = list() + + try: + git_grep_output = check_output(git_grep_command, universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + return git_grep_output + + +def main(): + exit_code = 0 + + regexp_ignore_known_violations = "|".join(KNOWN_VIOLATIONS) + git_grep_output = find_locale_dependent_function_uses() + + for locale_dependent_function in LOCALE_DEPENDENT_FUNCTIONS: + matches = [line for line in git_grep_output + if re.search("[^a-zA-Z0-9_\\`'\"<>]" + locale_dependent_function + "(_r|_s)?[^a-zA-Z0-9_\\`'\"<>]", line) + and not re.search("\\.(c|cpp|h):\\s*(//|\\*|/\\*|\").*" + locale_dependent_function, line) + and not re.search(regexp_ignore_known_violations, line)] + if matches: + print(f"The locale dependent function {locale_dependent_function}(...) appears to be used:") + for match in matches: + print(match) + print("") + exit_code = 1 + + if exit_code == 1: + print("Unnecessary locale depedence can cause bugs that are very tricky to isolate and fix. Please avoid using locale dependent functions if possible.\n") + print(f"Advice not applicable in this specific case? Add an exception by updating the ignore list in {sys.argv[0]}") + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh deleted file mode 100755 index 7d608eed6a..0000000000 --- a/test/lint/lint-locale-dependence.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2018-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -# Be aware that bitcoind and bitcoin-qt differ in terms of localization: Qt -# opts in to POSIX localization by running setlocale(LC_ALL, "") on startup, -# whereas no such call is made in bitcoind. -# -# Qt runs setlocale(LC_ALL, "") on initialization. This installs the locale -# specified by the user's LC_ALL (or LC_*) environment variable as the new -# C locale. -# -# In contrast, bitcoind does not opt in to localization -- no call to -# setlocale(LC_ALL, "") is made and the environment variables LC_* are -# thus ignored. -# -# This results in situations where bitcoind is guaranteed to be running -# with the classic locale ("C") whereas the locale of bitcoin-qt will vary -# depending on the user's environment variables. -# -# An example: Assuming the environment variable LC_ALL=de_DE then the -# call std::to_string(1.23) will return "1.230000" in bitcoind but -# "1,230000" in bitcoin-qt. -# -# From the Qt documentation: -# "On Unix/Linux Qt is configured to use the system locale settings by default. -# This can cause a conflict when using POSIX functions, for instance, when -# converting between data types such as floats and strings, since the notation -# may differ between locales. To get around this problem, call the POSIX function -# setlocale(LC_NUMERIC,"C") right after initializing QApplication, QGuiApplication -# or QCoreApplication to reset the locale that is used for number formatting to -# "C"-locale." -# -# See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and -# https://stackoverflow.com/a/34878283 for more details. - -# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. -KNOWN_VIOLATIONS=( - "src/dbwrapper.cpp:.*vsnprintf" - "src/test/dbwrapper_tests.cpp:.*snprintf" - "src/test/fuzz/locale.cpp" - "src/test/fuzz/string.cpp" - "src/test/util_tests.cpp" -) - -REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|minisketch/|tinyformat.h|univalue/)" - -LOCALE_DEPENDENT_FUNCTIONS=( - alphasort # LC_COLLATE (via strcoll) - asctime # LC_TIME (directly) - asprintf # (via vasprintf) - atof # LC_NUMERIC (via strtod) - atoi # LC_NUMERIC (via strtol) - atol # LC_NUMERIC (via strtol) - atoll # (via strtoll) - atoq - btowc # LC_CTYPE (directly) - ctime # (via asctime or localtime) - dprintf # (via vdprintf) - fgetwc - fgetws - fold_case # boost::locale::fold_case - fprintf # (via vfprintf) - fputwc - fputws - fscanf # (via __vfscanf) - fwprintf # (via __vfwprintf) - getdate # via __getdate_r => isspace // __localtime_r - getwc - getwchar - is_digit # boost::algorithm::is_digit - is_space # boost::algorithm::is_space - isalnum # LC_CTYPE - isalpha # LC_CTYPE - isblank # LC_CTYPE - iscntrl # LC_CTYPE - isctype # LC_CTYPE - isdigit # LC_CTYPE - isgraph # LC_CTYPE - islower # LC_CTYPE - isprint # LC_CTYPE - ispunct # LC_CTYPE - isspace # LC_CTYPE - isupper # LC_CTYPE - iswalnum # LC_CTYPE - iswalpha # LC_CTYPE - iswblank # LC_CTYPE - iswcntrl # LC_CTYPE - iswctype # LC_CTYPE - iswdigit # LC_CTYPE - iswgraph # LC_CTYPE - iswlower # LC_CTYPE - iswprint # LC_CTYPE - iswpunct # LC_CTYPE - iswspace # LC_CTYPE - iswupper # LC_CTYPE - iswxdigit # LC_CTYPE - isxdigit # LC_CTYPE - localeconv # LC_NUMERIC + LC_MONETARY - mblen # LC_CTYPE - mbrlen - mbrtowc - mbsinit - mbsnrtowcs - mbsrtowcs - mbstowcs # LC_CTYPE - mbtowc # LC_CTYPE - mktime - normalize # boost::locale::normalize - printf # LC_NUMERIC - putwc - putwchar - scanf # LC_NUMERIC - setlocale - snprintf - sprintf - sscanf - std::locale::global - std::to_string - stod - stof - stoi - stol - stold - stoll - stoul - stoull - strcasecmp - strcasestr - strcoll # LC_COLLATE -# strerror - strfmon - strftime # LC_TIME - strncasecmp - strptime - strtod # LC_NUMERIC - strtof - strtoimax - strtol # LC_NUMERIC - strtold - strtoll - strtoq - strtoul # LC_NUMERIC - strtoull - strtoumax - strtouq - strxfrm # LC_COLLATE - swprintf - to_lower # boost::locale::to_lower - to_title # boost::locale::to_title - to_upper # boost::locale::to_upper - tolower # LC_CTYPE - toupper # LC_CTYPE - towctrans - towlower # LC_CTYPE - towupper # LC_CTYPE - trim # boost::algorithm::trim - trim_left # boost::algorithm::trim_left - trim_right # boost::algorithm::trim_right - ungetwc - vasprintf - vdprintf - versionsort - vfprintf - vfscanf - vfwprintf - vprintf - vscanf - vsnprintf - vsprintf - vsscanf - vswprintf - vwprintf - wcrtomb - wcscasecmp - wcscoll # LC_COLLATE - wcsftime # LC_TIME - wcsncasecmp - wcsnrtombs - wcsrtombs - wcstod # LC_NUMERIC - wcstof - wcstoimax - wcstol # LC_NUMERIC - wcstold - wcstoll - wcstombs # LC_CTYPE - wcstoul # LC_NUMERIC - wcstoull - wcstoumax - wcswidth - wcsxfrm # LC_COLLATE - wctob - wctomb # LC_CTYPE - wctrans - wctype - wcwidth - wprintf -) - -function join_array { - local IFS="$1" - shift - echo "$*" -} - -REGEXP_IGNORE_KNOWN_VIOLATIONS=$(join_array "|" "${KNOWN_VIOLATIONS[@]}") - -# Invoke "git grep" only once in order to minimize run-time -REGEXP_LOCALE_DEPENDENT_FUNCTIONS=$(join_array "|" "${LOCALE_DEPENDENT_FUNCTIONS[@]}") -GIT_GREP_OUTPUT=$(git grep -E "[^a-zA-Z0-9_\`'\"<>](${REGEXP_LOCALE_DEPENDENT_FUNCTIONS}(_r|_s)?)[^a-zA-Z0-9_\`'\"<>]" -- "*.cpp" "*.h") - -EXIT_CODE=0 -for LOCALE_DEPENDENT_FUNCTION in "${LOCALE_DEPENDENT_FUNCTIONS[@]}"; do - MATCHES=$(grep -E "[^a-zA-Z0-9_\`'\"<>]${LOCALE_DEPENDENT_FUNCTION}(_r|_s)?[^a-zA-Z0-9_\`'\"<>]" <<< "${GIT_GREP_OUTPUT}" | \ - grep -vE "\.(c|cpp|h):\s*(//|\*|/\*|\").*${LOCALE_DEPENDENT_FUNCTION}") - if [[ ${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_EXTERNAL_DEPENDENCIES}" <<< "${MATCHES}") - fi - if [[ ${REGEXP_IGNORE_KNOWN_VIOLATIONS} != "" ]]; then - MATCHES=$(grep -vE "${REGEXP_IGNORE_KNOWN_VIOLATIONS}" <<< "${MATCHES}") - fi - if [[ ${MATCHES} != "" ]]; then - echo "The locale dependent function ${LOCALE_DEPENDENT_FUNCTION}(...) appears to be used:" - echo "${MATCHES}" - echo - EXIT_CODE=1 - fi -done -if [[ ${EXIT_CODE} != 0 ]]; then - echo "Unnecessary locale dependence can cause bugs that are very" - echo "tricky to isolate and fix. Please avoid using locale dependent" - echo "functions if possible." - echo - echo "Advice not applicable in this specific case? Add an exception" - echo "by updating the ignore list in $0" -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python-utf8-encoding.py b/test/lint/lint-python-utf8-encoding.py new file mode 100755 index 0000000000..62fdc34d50 --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +import sys +import re + +from subprocess import check_output, CalledProcessError + +EXCLUDED_DIRS = ["src/crc32c/"] + + +def get_exclude_args(): + return [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + +def check_fileopens(): + fileopens = list() + + try: + fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_fileopens = [fileopen for fileopen in fileopens if not re.search(r"encoding=.(ascii|utf8|utf-8).|open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]", fileopen)] + + return filtered_fileopens + + +def check_checked_outputs(): + checked_outputs = list() + + try: + checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"universal_newlines=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] + + return filtered_checked_outputs + + +def main(): + exit_code = 0 + + nonexplicit_utf8_fileopens = check_fileopens() + if nonexplicit_utf8_fileopens: + print("Python's open(...) seems to be used to open text files without explicitly specifying encoding='utf8':\n") + for fileopen in nonexplicit_utf8_fileopens: + print(fileopen) + exit_code = 1 + + nonexplicit_utf8_checked_outputs = check_checked_outputs() + if nonexplicit_utf8_checked_outputs: + if nonexplicit_utf8_fileopens: + print("\n") + print("Python's check_output(...) seems to be used to get program outputs without explicitly specifying encoding='utf8':\n") + for checked_output in nonexplicit_utf8_checked_outputs: + print(checked_output) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh deleted file mode 100755 index 6e5b18fc23..0000000000 --- a/test/lint/lint-python-utf8-encoding.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to -# avoid potential issues on the BSDs where the locale is not always set. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep " open(" -- "*.py" ":(exclude)src/crc32c/" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's open(...) seems to be used to open text files without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -OUTPUT=$(git grep "check_output(" -- "*.py" ":(exclude)src/crc32c/"| grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's check_output(...) seems to be used to get program outputs without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-shell-locale.py b/test/lint/lint-shell-locale.py new file mode 100755 index 0000000000..f3dfe18a95 --- /dev/null +++ b/test/lint/lint-shell-locale.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Make sure all shell scripts are: +a.) explicitly opt out of locale dependence using + "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or +b.) explicitly opt in to locale dependence using the annotation below. +""" + +import subprocess +import sys +import re + +OPT_IN_LINE = '# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"' + +OPT_OUT_LINES = [ + 'export LC_ALL=C', + 'export LC_ALL=C.UTF-8', +] + +def get_shell_files_list(): + command = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + shell_files = get_shell_files_list() + for file_path in shell_files: + if re.search('src/(secp256k1|minisketch|univalue)/', file_path): + continue + + with open(file_path, 'r', encoding='utf-8') as file_obj: + contents = file_obj.read() + + if OPT_IN_LINE in contents: + continue + + non_comment_pattern = re.compile(r'^\s*((?!#).+)$', re.MULTILINE) + non_comment_lines = re.findall(non_comment_pattern, contents) + if not non_comment_lines: + continue + + first_non_comment_line = non_comment_lines[0] + if first_non_comment_line not in OPT_OUT_LINES: + print(f'Missing "export LC_ALL=C" (to avoid locale dependence) as first non-comment non-empty line in {file_path}') + exit_code = 1 + + return sys.exit(exit_code) + +if __name__ == '__main__': + main() + diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh deleted file mode 100755 index 4c6b8a57e6..0000000000 --- a/test/lint/lint-shell-locale.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure all shell scripts: -# a.) explicitly opt out of locale dependence using -# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or -# b.) explicitly opt in to locale dependence using the annotation below. - -export LC_ALL=C - -EXIT_CODE=0 -for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|minisketch|univalue)/"); do - if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then - continue - fi - FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1) - if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then - echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}" - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-submodule.py b/test/lint/lint-submodule.py new file mode 100755 index 0000000000..89d4c80f55 --- /dev/null +++ b/test/lint/lint-submodule.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +This script checks for git modules +""" + +import subprocess +import sys + +def main(): + submodules_list = subprocess.check_output(['git', 'submodule', 'status', '--recursive'], + universal_newlines = True, encoding = 'utf8').rstrip('\n') + if submodules_list: + print("These submodules were found, delete them:\n", submodules_list) + sys.exit(1) + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-submodule.sh b/test/lint/lint-submodule.sh deleted file mode 100755 index d9aa021df7..0000000000 --- a/test/lint/lint-submodule.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script checks for git modules -export LC_ALL=C -EXIT_CODE=0 - -CMD=$(git submodule status --recursive) -if test -n "$CMD"; -then - echo These submodules were found, delete them: - echo "$CMD" - EXIT_CODE=1 -fi - -exit $EXIT_CODE - diff --git a/test/lint/lint-tests.py b/test/lint/lint-tests.py new file mode 100755 index 0000000000..849ddcb961 --- /dev/null +++ b/test/lint/lint-tests.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check the test suite naming conventions +""" + +import re +import subprocess +import sys + + +def grep_boost_fixture_test_suite(): + command = [ + "git", + "grep", + "-E", + r"^BOOST_FIXTURE_TEST_SUITE\(", + "--", + "src/test/**.cpp", + "src/wallet/test/**.cpp", + ] + return subprocess.check_output(command, universal_newlines=True, encoding="utf8") + + +def check_matching_test_names(test_suite_list): + not_matching = [ + x + for x in test_suite_list + if re.search(r"/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)", x) is None + ] + if len(not_matching) > 0: + not_matching = "\n".join(not_matching) + error_msg = ( + "The test suite in file src/test/foo_tests.cpp should be named\n" + '"foo_tests". Please make sure the following test suites follow\n' + "that convention:\n\n" + f"{not_matching}\n" + ) + print(error_msg) + return 1 + return 0 + + +def get_duplicates(input_list): + """ + From https://stackoverflow.com/a/9835819 + """ + seen = set() + dupes = set() + for x in input_list: + if x in seen: + dupes.add(x) + else: + seen.add(x) + return dupes + + +def check_unique_test_names(test_suite_list): + output = [re.search(r"\((.*?),", x) for x in test_suite_list] + output = [x.group(1) for x in output if x is not None] + output = get_duplicates(output) + output = sorted(list(output)) + + if len(output) > 0: + output = "\n".join(output) + error_msg = ( + "Test suite names must be unique. The following test suite names\n" + f"appear to be used more than once:\n\n{output}" + ) + print(error_msg) + return 1 + return 0 + + +def main(): + test_suite_list = grep_boost_fixture_test_suite().splitlines() + exit_code = check_matching_test_names(test_suite_list) + exit_code |= check_unique_test_names(test_suite_list) + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-tests.sh b/test/lint/lint-tests.sh deleted file mode 100755 index 35d11023eb..0000000000 --- a/test/lint/lint-tests.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check the test suite naming conventions - -export LC_ALL=C -EXIT_CODE=0 - -NAMING_INCONSISTENCIES=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ - "src/test/**.cpp" "src/wallet/test/**.cpp" | \ - grep -vE '/(.*?)\.cpp:BOOST_FIXTURE_TEST_SUITE\(\1, .*\)$') -if [[ ${NAMING_INCONSISTENCIES} != "" ]]; then - echo "The test suite in file src/test/foo_tests.cpp should be named" - echo "\"foo_tests\". Please make sure the following test suites follow" - echo "that convention:" - echo - echo "${NAMING_INCONSISTENCIES}" - EXIT_CODE=1 -fi - -TEST_SUITE_NAME_COLLISSIONS=$(git grep -E '^BOOST_FIXTURE_TEST_SUITE\(' -- \ - "src/test/**.cpp" "src/wallet/test/**.cpp" | cut -f2 -d'(' | cut -f1 -d, | \ - sort | uniq -d) -if [[ ${TEST_SUITE_NAME_COLLISSIONS} != "" ]]; then - echo "Test suite names must be unique. The following test suite names" - echo "appear to be used more than once:" - echo - echo "${TEST_SUITE_NAME_COLLISSIONS}" - EXIT_CODE=1 -fi - -exit ${EXIT_CODE} |