diff options
62 files changed, 1119 insertions, 308 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 31c3aba8c6..d4a343e777 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,8 +11,8 @@ environment: PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 cache: -- C:\tools\vcpkg\installed -- C:\Users\appveyor\clcache +- C:\tools\vcpkg\installed -> appveyor.yml +- C:\Users\appveyor\clcache -> appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in install: - cmd: pip install --quiet git+https://github.com/frerich/clcache.git@v4.2.0 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. diff --git a/.travis.yml b/.travis.yml index d5086b084a..0ede31b4a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ env: - MAKEJOBS=-j3 - RUN_UNIT_TESTS=true - RUN_FUNCTIONAL_TESTS=true + - RUN_FUZZ_TESTS=false - DOCKER_NAME_TAG=ubuntu:18.04 - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID - CCACHE_SIZE=100M @@ -85,12 +86,12 @@ jobs: BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" - stage: test - name: '32-bit + dash [GOAL: install]' + name: '32-bit + dash [GOAL: install] [GUI: no BIP70]' env: >- HOST=i686-pc-linux-gnu PACKAGES="g++-multilib python3-zmq" GOAL="install" - BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" + BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --disable-bip70 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" CONFIG_SHELL="/bin/dash" - stage: test @@ -100,7 +101,7 @@ jobs: PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" GOAL="install" - BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-fuzz --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\"" + BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\"" - stage: test name: 'x86_64 Linux [GOAL: install] [trusty] [no functional tests, no depends, only system libs]' @@ -133,6 +134,18 @@ jobs: BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" - stage: test + name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: fuzzer,address]' + env: >- + HOST=x86_64-unknown-linux-gnu + PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev" + NO_DEPENDS=1 + RUN_UNIT_TESTS=false + RUN_FUNCTIONAL_TESTS=false + RUN_FUZZ_TESTS=true + GOAL="install" + BITCOIN_CONFIG="--disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++" + + - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]' env: >- HOST=x86_64-unknown-linux-gnu diff --git a/.travis/test_04_install.sh b/.travis/test_04_install.sh index a111387f10..8055bbdd19 100755 --- a/.travis/test_04_install.sh +++ b/.travis/test_04_install.sh @@ -7,6 +7,11 @@ export LC_ALL=C.UTF-8 travis_retry docker pull "$DOCKER_NAME_TAG" + +export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets +git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN} +export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/ + mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/" export ASAN_OPTIONS="" export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan" diff --git a/.travis/test_06_script_b.sh b/.travis/test_06_script_b.sh index b6b5925be8..e13abfd52f 100755 --- a/.travis/test_06_script_b.sh +++ b/.travis/test_06_script_b.sh @@ -19,3 +19,9 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then DOCKER_EXEC test/functional/test_runner.py --ci --combinedlogslen=4000 --coverage --quiet --failfast END_FOLD fi + +if [ "$RUN_FUZZ_TESTS" = "true" ]; then + BEGIN_FOLD fuzz-tests + DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN} + END_FOLD +fi diff --git a/Makefile.am b/Makefile.am index 9a6e15f0dd..85674f819a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -20,6 +20,7 @@ BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT) BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT) BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT) BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT) +BITCOIN_WALLET_BIN=$(top_builddir)/src/$(BITCOIN_WALLET_TOOL_NAME)$(EXEEXT) BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win$(WINDOWS_BITS)-setup$(EXEEXT) empty := @@ -76,6 +77,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release + STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release @test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \ echo error: could not build $@ @echo built $@ @@ -172,6 +174,9 @@ $(BITCOIN_CLI_BIN): FORCE $(BITCOIN_TX_BIN): FORCE $(MAKE) -C src $(@F) +$(BITCOIN_WALLET_BIN): FORCE + $(MAKE) -C src $(@F) + if USE_LCOV LCOV_FILTER_PATTERN=-p "/usr/include/" -p "/usr/lib/" -p "src/leveldb/" -p "src/bench/" -p "src/univalue" -p "src/crypto/ctaes" -p "src/secp256k1" @@ -220,7 +225,11 @@ endif dist_noinst_SCRIPTS = autogen.sh -EXTRA_DIST = $(DIST_SHARE) test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) +EXTRA_DIST = $(DIST_SHARE) $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) + +EXTRA_DIST += \ + test/functional \ + test/fuzz EXTRA_DIST += \ test/util/bitcoin-util-test.py \ diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 90a2cd9875..1a7c5d5f7d 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -116,24 +116,6 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ if test "x$bitcoin_cv_static_qt" = xyes; then _BITCOIN_QT_FIND_STATIC_PLUGINS AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static]) - AC_CACHE_CHECK(for Qt < 5.4, bitcoin_cv_need_acc_widget,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include <QtCore/qconfig.h> - #ifndef QT_VERSION - # include <QtCore/qglobal.h> - #endif - ]], - [[ - #if QT_VERSION >= 0x050400 - choke - #endif - ]])], - [bitcoin_cv_need_acc_widget=yes], - [bitcoin_cv_need_acc_widget=no]) - ]) - if test "x$bitcoin_cv_need_acc_widget" = xyes; then - _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(AccessibleFactory)], [-lqtaccessiblewidgets]) - fi _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) if test "x$TARGET_OS" = xwindows; then @@ -264,7 +246,7 @@ dnl All macros below are internal and should _not_ be used from the main dnl configure.ac. dnl ---- -dnl Internal. Check if the included version of Qt is Qt5. +dnl Internal. Check included version of Qt against minimum specified in doc/dependencies.md dnl Requires: INCLUDES must be populated as necessary. dnl Output: bitcoin_cv_qt5=yes|no AC_DEFUN([_BITCOIN_QT_CHECK_QT5],[ @@ -276,7 +258,7 @@ AC_DEFUN([_BITCOIN_QT_CHECK_QT5],[ #endif ]], [[ - #if QT_VERSION < 0x050200 || QT_VERSION_MAJOR < 5 + #if QT_VERSION < 0x050501 choke #endif ]])], @@ -374,9 +356,7 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ fi if test "x$TARGET_OS" = xlinux; then PKG_CHECK_MODULES([X11XCB], [x11-xcb], [QT_LIBS="$X11XCB_LIBS $QT_LIBS"]) - if ${PKG_CONFIG} --exists "Qt5Core >= 5.5" 2>/dev/null; then - PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) - fi + PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) elif test "x$TARGET_OS" = xdarwin; then PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) @@ -527,4 +507,3 @@ AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG],[ CXXFLAGS="$TEMP_CXXFLAGS" LIBS="$TEMP_LIBS" ]) - diff --git a/configure.ac b/configure.ac index ddd6c7f8eb..dec6d4fac3 100644 --- a/configure.ac +++ b/configure.ac @@ -85,8 +85,8 @@ AC_PATH_TOOL(RANLIB, ranlib) AC_PATH_TOOL(STRIP, strip) AC_PATH_TOOL(GCOV, gcov) AC_PATH_PROG(LCOV, lcov) -dnl Python 3.x is supported from 3.4 on (see https://github.com/bitcoin/bitcoin/issues/7893) -AC_PATH_PROGS([PYTHON], [python3.7 python3.6 python3.5 python3.4 python3 python]) +dnl Python 3.4 is specified in .python-version and should be used if available, see doc/dependencies.md +AC_PATH_PROGS([PYTHON], [python3.4 python3.5 python3.6 python3.7 python3 python]) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) diff --git a/doc/dependencies.md b/doc/dependencies.md index b833e9151f..235eeb28b6 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -23,7 +23,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | protobuf | [2.6.1](https://github.com/google/protobuf/releases) | | No | | | | Python (tests) | | [3.4](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.2](https://github.com/bitcoin/bitcoin/pull/14725) | No | | | +| Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L87) (Linux only) | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L86) (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 08b73d3b3c..f9221dde5b 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -5,6 +5,29 @@ A special test harness in `src/test/fuzz/` is provided for each fuzz target to provide an easy entry point for fuzzers and the like. In this document we'll describe how to use it with AFL and libFuzzer. +## Preparing fuzzing + +AFL needs an input directory with examples, and an output directory where it +will place examples that it found. These can be anywhere in the file system, +we'll define environment variables to make it easy to reference them. + +libFuzzer will use the input directory as output directory. + +Extract the example seeds (or other starting inputs) into the inputs +directory before starting fuzzing. + +``` +git clone https://github.com/bitcoin-core/qa-assets +export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus +``` + +Only for AFL: + +``` +mkdir outputs +export AFLOUT=$PWD/outputs +``` + ## AFL ### Building AFL @@ -23,7 +46,7 @@ export AFLPATH=$PWD To build Bitcoin Core using AFL instrumentation (this assumes that the `AFLPATH` was set as above): ``` -./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++ +./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++ export AFL_HARDEN=1 cd src/ make @@ -39,31 +62,14 @@ binary will be instrumented in such a way that the AFL features "persistent mode" and "deferred forkserver" can be used. See https://github.com/mcarpenter/afl/tree/master/llvm_mode for details. -### Preparing fuzzing - -AFL needs an input directory with examples, and an output directory where it -will place examples that it found. These can be anywhere in the file system, -we'll define environment variables to make it easy to reference them. - -``` -mkdir inputs -AFLIN=$PWD/inputs -mkdir outputs -AFLOUT=$PWD/outputs -``` - -Example inputs are available from: - -- https://download.visucore.com/bitcoin/bitcoin_fuzzy_in.tar.xz -- http://strateman.ninja/fuzzing.tar.xz - -Extract these (or other starting inputs) into the `inputs` directory before starting fuzzing. - ### Fuzzing To start the actual fuzzing use: + ``` -$AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/fuzz/fuzz_target_foo +export FUZZ_TARGET=fuzz_target_foo # Pick a fuzz_target +mkdir ${AFLOUT}/${FUZZ_TARGET} +$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- test/fuzz/${FUZZ_TARGET} ``` You may have to change a few kernel parameters to test optimally - `afl-fuzz` @@ -74,10 +80,10 @@ will print an error and suggestion if so. A recent version of `clang`, the address sanitizer and libFuzzer is needed (all found in the `compiler-rt` runtime libraries package). -To build the `test/test_bitcoin_fuzzy` executable run +To build all fuzz targets with libFuzzer, run ``` -./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++ +./configure --disable-ccache --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++ make ``` @@ -86,3 +92,6 @@ interchangeably between libFuzzer and AFL. See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer instrumented executable. + +Alternatively run the script in `./test/fuzz/test_runner.py` and provide it +with the `${DIR_FUZZ_IN}` created earlier. diff --git a/doc/release-notes-14021.md b/doc/release-notes-14021.md new file mode 100644 index 0000000000..4797a95bdb --- /dev/null +++ b/doc/release-notes-14021.md @@ -0,0 +1,11 @@ +Miscellaneous RPC Changes +------------------------- +- Descriptors with key origin information imported through `importmulti` will have their key origin information stored in the wallet for use with creating PSBTs. +- If `bip32derivs` of both `walletprocesspsbt` and `walletcreatefundedpsbt` is set to true but the key metadata for a public key has not been updated yet, then that key will have a derivation path as if it were just an independent key (i.e. no derivation path and its master fingerprint is itself) + +Miscellaneous Wallet changes +---------------------------- + +- The key metadata will need to be upgraded the first time that the HD seed is available. +For unencrypted wallets this will occur on wallet loading. +For encrypted wallets this will occur the first time the wallet is unlocked. diff --git a/doc/release-notes-14481.md b/doc/release-notes-14481.md new file mode 100644 index 0000000000..ea8fc3c34e --- /dev/null +++ b/doc/release-notes-14481.md @@ -0,0 +1,9 @@ +Low-level RPC changes +---------------------- + +The `listunspent` RPC has been modified so that it also returns `witnessScript`, +the witness script in the case of a P2WSH or P2SH-P2WSH output. + +The `signrawtransactionwithkey` and `signrawtransactionwithwallet` RPCs have been +modified so that they also optionally accept a `witnessScript`, the witness script in the +case of a P2WSH or P2SH-P2WSH output. This is compatible with the change to `listunspent`. diff --git a/doc/release-notes-15393.md b/doc/release-notes-15393.md new file mode 100644 index 0000000000..f478dc798d --- /dev/null +++ b/doc/release-notes-15393.md @@ -0,0 +1,4 @@ +Dependencies +------------ + +- The minimum required version of QT has been increased from 5.2 to 5.5.1 (the [depends system](https://github.com/bitcoin/bitcoin/blob/master/depends/README.md) provides 5.9.7) diff --git a/doc/release-notes.md b/doc/release-notes.md index 113b8c07d0..a6408cf1e6 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -263,6 +263,9 @@ in the Low-level Changes section below. - See the [Mining](#mining) section for changes to `getblocktemplate`. +- The `getmininginfo` RPC now omits `currentblockweight` and `currentblocktx` + when a block was never assembled via RPC on this node. + - The `getrawtransaction` RPC & REST endpoints no longer check the unspent UTXO set for a transaction. The remaining behaviors are as follows: 1. If a blockhash is provided, check the corresponding block. diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 6542370f97..046deed9ea 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -81,6 +81,7 @@ Section -Main SEC0000 File @abs_top_srcdir@/release/@BITCOIN_DAEMON_NAME@@EXEEXT@ File @abs_top_srcdir@/release/@BITCOIN_CLI_NAME@@EXEEXT@ File @abs_top_srcdir@/release/@BITCOIN_TX_NAME@@EXEEXT@ + File @abs_top_srcdir@/release/@BITCOIN_WALLET_TOOL_NAME@@EXEEXT@ SetOutPath $INSTDIR\doc File /r /x Makefile* @abs_top_srcdir@/doc\*.* SetOutPath $INSTDIR diff --git a/src/Makefile.am b/src/Makefile.am index 7490d8b790..d491530ca1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -118,6 +118,7 @@ BITCOIN_CORE_H = \ clientversion.h \ coins.h \ compat.h \ + compat/assumptions.h \ compat/byteswap.h \ compat/endian.h \ compat/sanity.h \ @@ -196,6 +197,7 @@ BITCOIN_CORE_H = \ txmempool.h \ ui_interface.h \ undo.h \ + util/bip32.h \ util/bytevectorhash.h \ util/system.h \ util/memory.h \ @@ -456,6 +458,7 @@ libbitcoin_util_a_SOURCES = \ support/cleanse.cpp \ sync.cpp \ threadinterrupt.cpp \ + util/bip32.cpp \ util/bytevectorhash.cpp \ util/system.cpp \ util/moneystr.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 12112375ea..84bc326cfe 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -2,7 +2,6 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -bin_PROGRAMS += test/test_bitcoin FUZZ_TARGETS = \ test/fuzz/address_deserialize \ @@ -21,6 +20,7 @@ FUZZ_TARGETS = \ test/fuzz/inv_deserialize \ test/fuzz/messageheader_deserialize \ test/fuzz/netaddr_deserialize \ + test/fuzz/script_flags \ test/fuzz/service_deserialize \ test/fuzz/transaction_deserialize \ test/fuzz/txoutcompressor_deserialize \ @@ -28,6 +28,8 @@ FUZZ_TARGETS = \ if ENABLE_FUZZ noinst_PROGRAMS += $(FUZZ_TARGETS:=) +else +bin_PROGRAMS += test/test_bitcoin endif TEST_SRCDIR = test @@ -171,7 +173,7 @@ test_test_bitcoin_LDADD += $(ZMQ_LIBS) endif if ENABLE_FUZZ -test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_DESERIALIZE=1 test_fuzz_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -188,7 +190,7 @@ test_fuzz_block_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_block_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTRANSACTION_DESERIALIZE=1 test_fuzz_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -205,7 +207,7 @@ test_fuzz_transaction_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_transaction_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocklocator_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKLOCATOR_DESERIALIZE=1 test_fuzz_blocklocator_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocklocator_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -222,7 +224,7 @@ test_fuzz_blocklocator_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_blocklocator_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockmerkleroot_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKMERKLEROOT=1 test_fuzz_blockmerkleroot_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockmerkleroot_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -239,7 +241,7 @@ test_fuzz_blockmerkleroot_LDADD = \ $(LIBSECP256K1) test_fuzz_blockmerkleroot_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_addrman_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRMAN_DESERIALIZE=1 test_fuzz_addrman_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addrman_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -256,7 +258,7 @@ test_fuzz_addrman_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_addrman_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -273,7 +275,7 @@ test_fuzz_blockheader_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_blockheader_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -290,7 +292,7 @@ test_fuzz_banentry_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_banentry_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -307,7 +309,7 @@ test_fuzz_txundo_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_txundo_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKUNDO_DESERIALIZE=1 test_fuzz_blockundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -324,7 +326,7 @@ test_fuzz_blockundo_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_blockundo_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -341,7 +343,7 @@ test_fuzz_coins_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_coins_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -358,7 +360,24 @@ test_fuzz_netaddr_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_netaddr_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_script_flags_SOURCES = $(FUZZ_SUITE) test/fuzz/script_flags.cpp +test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_flags_LDADD = \ + $(LIBUNIVALUE) \ + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBBITCOIN_CRYPTO_SSE41) \ + $(LIBBITCOIN_CRYPTO_AVX2) \ + $(LIBBITCOIN_CRYPTO_SHANI) \ + $(LIBSECP256K1) +test_fuzz_script_flags_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) + +test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_service_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSERVICE_DESERIALIZE=1 test_fuzz_service_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_service_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -375,7 +394,7 @@ test_fuzz_service_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_service_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_messageheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -392,7 +411,7 @@ test_fuzz_messageheader_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_messageheader_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_address_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -409,7 +428,7 @@ test_fuzz_address_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_address_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_inv_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -426,7 +445,7 @@ test_fuzz_inv_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_inv_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_bloomfilter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOOMFILTER_DESERIALIZE=1 test_fuzz_bloomfilter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -443,7 +462,7 @@ test_fuzz_bloomfilter_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_bloomfilter_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_diskblockindex_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DDISKBLOCKINDEX_DESERIALIZE=1 test_fuzz_diskblockindex_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_diskblockindex_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -460,7 +479,7 @@ test_fuzz_diskblockindex_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_diskblockindex_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txoutcompressor_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXOUTCOMPRESSOR_DESERIALIZE=1 test_fuzz_txoutcompressor_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -477,7 +496,7 @@ test_fuzz_txoutcompressor_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_txoutcompressor_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactions_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) @@ -494,7 +513,7 @@ test_fuzz_blocktransactions_deserialize_LDADD = \ $(LIBSECP256K1) test_fuzz_blocktransactions_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) -test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 8133fd9d65..cd4543c012 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -73,10 +73,12 @@ static void AssembleBlock(benchmark::State& state) boost::thread_group thread_group; CScheduler scheduler; { + LOCK(cs_main); ::pblocktree.reset(new CBlockTreeDB(1 << 20, true)); ::pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); ::pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - + } + { const CChainParams& chainparams = Params(); thread_group.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index e0854e2c62..1f6840d813 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -37,10 +37,12 @@ static void DuplicateInputs(benchmark::State& state) CScheduler scheduler; const CChainParams& chainparams = Params(); { + LOCK(cs_main); ::pblocktree.reset(new CBlockTreeDB(1 << 20, true)); ::pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); ::pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - + } + { thread_group.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); LoadGenesisBlock(chainparams); diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h new file mode 100644 index 0000000000..820c9b93d9 --- /dev/null +++ b/src/compat/assumptions.h @@ -0,0 +1,49 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Compile-time verification of assumptions we make. + +#ifndef BITCOIN_COMPAT_ASSUMPTIONS_H +#define BITCOIN_COMPAT_ASSUMPTIONS_H + +#include <limits> + +// Assumption: We assume that the macro NDEBUG is not defined. +// Example(s): We use assert(...) extensively with the assumption of it never +// being a noop at runtime. +#if defined(NDEBUG) +# error "Bitcoin cannot be compiled without assertions." +#endif + +// Assumption: We assume the floating-point types to fulfill the requirements of +// IEC 559 (IEEE 754) standard. +// Example(s): Floating-point division by zero in ConnectBlock, CreateTransaction +// and EstimateMedianVal. +static_assert(std::numeric_limits<float>::is_iec559, "IEEE 754 float assumed"); +static_assert(std::numeric_limits<double>::is_iec559, "IEEE 754 double assumed"); + +// Assumption: We assume eight bits per byte (obviously, but remember: don't +// trust -- verify!). +// Example(s): Everywhere :-) +static_assert(std::numeric_limits<unsigned char>::digits == 8, "8-bit byte assumed"); + +// Assumption: We assume floating-point widths. +// Example(s): Type punning in serialization code (ser_{float,double}_to_uint{32,64}). +static_assert(sizeof(float) == 4, "32-bit float assumed"); +static_assert(sizeof(double) == 8, "64-bit double assumed"); + +// Assumption: We assume integer widths. +// Example(s): GetSizeOfCompactSize and WriteCompactSize in the serialization +// code. +static_assert(sizeof(short) == 2, "16-bit short assumed"); +static_assert(sizeof(int) == 4, "32-bit int assumed"); + +// Some important things we are NOT assuming (non-exhaustive list): +// * We are NOT assuming a specific value for sizeof(std::size_t). +// * We are NOT assuming a specific value for std::endian::native. +// * We are NOT assuming a specific value for std::locale("").name(). +// * We are NOT assuming a specific value for std::numeric_limits<char>::is_signed. + +#endif // BITCOIN_COMPAT_ASSUMPTIONS_H diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index 206227b101..5b5430037c 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -468,6 +468,10 @@ public: bool IsWalletFlagSet(uint64_t flag) override { return m_wallet.IsWalletFlagSet(flag); } OutputType getDefaultAddressType() override { return m_wallet.m_default_address_type; } OutputType getDefaultChangeType() override { return m_wallet.m_default_change_type; } + void remove() override + { + RemoveWallet(m_shared_wallet); + } std::unique_ptr<Handler> handleUnload(UnloadFn fn) override { return MakeHandler(m_wallet.NotifyUnload.connect(fn)); diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index a86212356c..a931e5fafb 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -247,6 +247,9 @@ public: // Get default change type. virtual OutputType getDefaultChangeType() = 0; + // Remove wallet. + virtual void remove() = 0; + //! Register handler for unload message. using UnloadFn = std::function<void()>; virtual std::unique_ptr<Handler> handleUnload(UnloadFn fn) = 0; diff --git a/src/miner.cpp b/src/miner.cpp index ef48a86e32..80a2f8f018 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,8 +10,8 @@ #include <chainparams.h> #include <coins.h> #include <consensus/consensus.h> -#include <consensus/tx_verify.h> #include <consensus/merkle.h> +#include <consensus/tx_verify.h> #include <consensus/validation.h> #include <hash.h> #include <net.h> @@ -21,22 +21,14 @@ #include <primitives/transaction.h> #include <script/standard.h> #include <timedata.h> -#include <util/system.h> #include <util/moneystr.h> +#include <util/system.h> #include <validationinterface.h> #include <algorithm> #include <queue> #include <utility> -// Unconfirmed transactions in the memory pool often depend on other -// transactions in the memory pool. When we select transactions from the -// pool, we select by highest fee rate of a transaction combined with all -// its ancestors. - -uint64_t nLastBlockTx = 0; -uint64_t nLastBlockWeight = 0; - int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { int64_t nOldTime = pblock->nTime; @@ -95,6 +87,9 @@ void BlockAssembler::resetBlock() nFees = 0; } +Optional<int64_t> BlockAssembler::m_last_block_num_txs{nullopt}; +Optional<int64_t> BlockAssembler::m_last_block_weight{nullopt}; + std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn) { int64_t nTimeStart = GetTimeMicros(); @@ -147,8 +142,8 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc int64_t nTime1 = GetTimeMicros(); - nLastBlockTx = nBlockTx; - nLastBlockWeight = nBlockWeight; + m_last_block_num_txs = nBlockTx; + m_last_block_weight = nBlockWeight; // Create coinbase transaction. CMutableTransaction coinbaseTx; diff --git a/src/miner.h b/src/miner.h index 44c50b01ad..7c4c455072 100644 --- a/src/miner.h +++ b/src/miner.h @@ -1,17 +1,19 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_MINER_H #define BITCOIN_MINER_H +#include <optional.h> #include <primitives/block.h> #include <txmempool.h> #include <validation.h> -#include <stdint.h> #include <memory> +#include <stdint.h> + #include <boost/multi_index_container.hpp> #include <boost/multi_index/ordered_index.hpp> @@ -159,6 +161,9 @@ public: /** Construct a new block template with coinbase to scriptPubKeyIn */ std::unique_ptr<CBlockTemplate> CreateNewBlock(const CScript& scriptPubKeyIn); + static Optional<int64_t> m_last_block_num_txs; + static Optional<int64_t> m_last_block_weight; + private: // utility functions /** Clear the block's state and prepare for assembling a new block */ diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 5927a14a6e..18dd2f010c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -85,6 +85,7 @@ struct COrphanTx { CTransactionRef tx; NodeId fromPeer; int64_t nTimeExpire; + size_t list_pos; }; CCriticalSection g_cs_orphans; std::map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(g_cs_orphans); @@ -186,6 +187,8 @@ namespace { }; std::map<COutPoint, std::set<std::map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(g_cs_orphans); + std::vector<std::map<uint256, COrphanTx>::iterator> g_orphan_list GUARDED_BY(g_cs_orphans); //! For random eviction + static size_t vExtraTxnForCompactIt GUARDED_BY(g_cs_orphans) = 0; static std::vector<std::pair<uint256, CTransactionRef>> vExtraTxnForCompact GUARDED_BY(g_cs_orphans); } // namespace @@ -837,8 +840,9 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE return false; } - auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME}); + auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME, g_orphan_list.size()}); assert(ret.second); + g_orphan_list.push_back(ret.first); for (const CTxIn& txin : tx->vin) { mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); } @@ -864,6 +868,18 @@ int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) if (itPrev->second.empty()) mapOrphanTransactionsByPrev.erase(itPrev); } + + size_t old_pos = it->second.list_pos; + assert(g_orphan_list[old_pos] == it); + if (old_pos + 1 != g_orphan_list.size()) { + // Unless we're deleting the last entry in g_orphan_list, move the last + // entry to the position we're deleting. + auto it_last = g_orphan_list.back(); + g_orphan_list[old_pos] = it_last; + it_last->second.list_pos = old_pos; + } + g_orphan_list.pop_back(); + mapOrphanTransactions.erase(it); return 1; } @@ -914,11 +930,8 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) while (mapOrphanTransactions.size() > nMaxOrphans) { // Evict a random orphan: - uint256 randomhash = rng.rand256(); - std::map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.lower_bound(randomhash); - if (it == mapOrphanTransactions.end()) - it = mapOrphanTransactions.begin(); - EraseOrphanTx(it->first); + size_t randompos = rng.randrange(g_orphan_list.size()); + EraseOrphanTx(g_orphan_list[randompos]->first); ++nEvicted; } return nEvicted; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index d6c6fd6e98..1b063771ef 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -55,9 +55,6 @@ #if defined(QT_STATICPLUGIN) #include <QtPlugin> -#if QT_VERSION < 0x050400 -Q_IMPORT_PLUGIN(AccessibleFactory) -#endif #if defined(QT_QPA_PLATFORM_XCB) Q_IMPORT_PLUGIN(QXcbIntegrationPlugin); #elif defined(QT_QPA_PLATFORM_WINDOWS) diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index f7a4bad916..bc88dd5e0a 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -338,6 +338,9 @@ void BitcoinGUI::createActions() m_open_wallet_action->setMenu(new QMenu(this)); m_open_wallet_action->setStatusTip(tr("Open a wallet")); + m_close_wallet_action = new QAction(tr("Close Wallet..."), this); + m_close_wallet_action->setStatusTip(tr("Close wallet")); + showHelpMessageAction = new QAction(platformStyle->TextColorIcon(":/icons/info"), tr("&Command-line options"), this); showHelpMessageAction->setMenuRole(QAction::NoRole); showHelpMessageAction->setStatusTip(tr("Show the %1 help message to get a list with possible Bitcoin command-line options").arg(tr(PACKAGE_NAME))); @@ -396,6 +399,9 @@ void BitcoinGUI::createActions() }); } }); + connect(m_close_wallet_action, &QAction::triggered, [this] { + m_wallet_controller->closeWallet(walletFrame->currentWalletModel(), this); + }); } #endif // ENABLE_WALLET @@ -418,6 +424,7 @@ void BitcoinGUI::createMenuBar() if(walletFrame) { file->addAction(m_open_wallet_action); + file->addAction(m_close_wallet_action); file->addSeparator(); file->addAction(openAction); file->addAction(backupWalletAction); @@ -693,6 +700,7 @@ void BitcoinGUI::setWalletActionsEnabled(bool enabled) usedSendingAddressesAction->setEnabled(enabled); usedReceivingAddressesAction->setEnabled(enabled); openAction->setEnabled(enabled); + m_close_wallet_action->setEnabled(enabled); } void BitcoinGUI::createTrayIcon() diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index c226494020..b58ccbb455 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -148,6 +148,7 @@ private: QAction* openAction = nullptr; QAction* showHelpMessageAction = nullptr; QAction* m_open_wallet_action{nullptr}; + QAction* m_close_wallet_action{nullptr}; QAction* m_wallet_selector_label_action = nullptr; QAction* m_wallet_selector_action = nullptr; diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index b1ec90ad36..43dccec4ea 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -292,9 +292,9 @@ void PaymentServer::handleURIOrFile(const QString& s) else if (s.startsWith(BITCOIN_IPC_PREFIX, Qt::CaseInsensitive)) // bitcoin: URI { QUrlQuery uri((QUrl(s))); +#ifdef ENABLE_BIP70 if (uri.hasQueryItem("r")) // payment request URI { -#ifdef ENABLE_BIP70 Q_EMIT message(tr("URI handling"), tr("You are using a BIP70 URL which will be unsupported in the future."), CClientUIInterface::ICON_WARNING); @@ -315,19 +315,23 @@ void PaymentServer::handleURIOrFile(const QString& s) tr("Payment request fetch URL is invalid: %1").arg(fetchUrl.toString()), CClientUIInterface::ICON_WARNING); } -#else - Q_EMIT message(tr("URI handling"), - tr("Cannot process payment request because BIP70 support was not compiled in."), - CClientUIInterface::ICON_WARNING); -#endif return; } - else // normal URI + else +#endif + // normal URI { SendCoinsRecipient recipient; if (GUIUtil::parseBitcoinURI(s, &recipient)) { if (!IsValidDestinationString(recipient.address.toStdString())) { +#ifndef ENABLE_BIP70 + if (uri.hasQueryItem("r")) { // payment request + Q_EMIT message(tr("URI handling"), + tr("Cannot process payment request because BIP70 support was not compiled in."), + CClientUIInterface::ICON_WARNING); + } +#endif Q_EMIT message(tr("URI handling"), tr("Invalid payment address %1").arg(recipient.address), CClientUIInterface::MSG_ERROR); } @@ -343,9 +347,9 @@ void PaymentServer::handleURIOrFile(const QString& s) } } -#ifdef ENABLE_BIP70 if (QFile::exists(s)) // payment request file { +#ifdef ENABLE_BIP70 PaymentRequestPlus request; SendCoinsRecipient recipient; if (!readPaymentRequestFromFile(s, request)) @@ -358,8 +362,12 @@ void PaymentServer::handleURIOrFile(const QString& s) Q_EMIT receivedPaymentRequest(recipient); return; - } +#else + Q_EMIT message(tr("Payment request file handling"), + tr("Cannot process payment request because BIP70 support was not compiled in."), + CClientUIInterface::ICON_WARNING); #endif + } } void PaymentServer::handleURIConnection() diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 2c477a2e98..da25d83175 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -27,9 +27,7 @@ #include <QTest> #include <QTextEdit> #include <QtGlobal> -#if QT_VERSION >= 0x050000 #include <QtTest/QtTestWidgets> -#endif #include <QtTest/QtTestGui> #include <new> #include <string> diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 173c814f1e..f8a9c25303 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -120,7 +120,6 @@ void RPCNestedTests::rpcNestedTests() RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest( abc , cba )"); QVERIFY(result == "[\"abc\",\"cba\"]"); -#if QT_VERSION >= 0x050300 // do the QVERIFY_EXCEPTION_THROWN checks only with Qt5.3 and higher (QVERIFY_EXCEPTION_THROWN was introduced in Qt5.3) QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo() .\n"), std::runtime_error); //invalid syntax QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo() getblockchaininfo()"), std::runtime_error); //invalid syntax @@ -131,5 +130,4 @@ void RPCNestedTests::rpcNestedTests() QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest abc,,abc"), std::runtime_error); //don't tollerate empty arguments when using , QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest(abc,,abc)"), std::runtime_error); //don't tollerate empty arguments when using , QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest(abc,,)"), std::runtime_error); //don't tollerate empty arguments when using , -#endif } diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 3483c75970..c532ffbbfe 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -63,6 +63,22 @@ OpenWalletActivity* WalletController::openWallet(const std::string& name, QWidge return activity; } +void WalletController::closeWallet(WalletModel* wallet_model, QWidget* parent) +{ + QMessageBox box(parent); + box.setWindowTitle(tr("Close wallet")); + box.setText(tr("Are you sure you wish to close wallet <i>%1</i>?").arg(wallet_model->getDisplayName())); + box.setInformativeText(tr("Closing the wallet for too long can result in having to resync the entire chain if pruning is enabled.")); + box.setStandardButtons(QMessageBox::Yes|QMessageBox::Cancel); + box.setDefaultButton(QMessageBox::Yes); + if (box.exec() != QMessageBox::Yes) return; + + // First remove wallet from node. + wallet_model->wallet().remove(); + // Now release the model. + removeAndDeleteWallet(wallet_model); +} + WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wallet> wallet) { QMutexLocker locker(&m_mutex); diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index f19c0e1f3d..19b3a82253 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -44,6 +44,7 @@ public: std::vector<std::string> getWalletsAvailableToOpen() const; OpenWalletActivity* openWallet(const std::string& name, QWidget* parent = nullptr); + void closeWallet(WalletModel* wallet_model, QWidget* parent = nullptr); private Q_SLOTS: void addWallet(WalletModel* wallet_model); diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 66a3b1048e..6625a03bbd 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -187,36 +187,36 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) static UniValue getmininginfo(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 0) + if (request.fHelp || request.params.size() != 0) { throw std::runtime_error( RPCHelpMan{"getmininginfo", "\nReturns a json object containing mining-related information.", {}, RPCResult{ - "{\n" - " \"blocks\": nnn, (numeric) The current block\n" - " \"currentblockweight\": nnn, (numeric) The last block weight\n" - " \"currentblocktx\": nnn, (numeric) The last block transaction\n" - " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" - " \"networkhashps\": nnn, (numeric) The network hashes per second\n" - " \"pooledtx\": n (numeric) The size of the mempool\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" - " \"warnings\": \"...\" (string) any network and blockchain warnings\n" - "}\n" + "{\n" + " \"blocks\": nnn, (numeric) The current block\n" + " \"currentblockweight\": nnn, (numeric, optional) The block weight of the last assembled block (only present if a block was ever assembled)\n" + " \"currentblocktx\": nnn, (numeric, optional) The number of block transactions of the last assembled block (only present if a block was ever assembled)\n" + " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" + " \"networkhashps\": nnn, (numeric) The network hashes per second\n" + " \"pooledtx\": n (numeric) The size of the mempool\n" + " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"warnings\": \"...\" (string) any network and blockchain warnings\n" + "}\n" }, RPCExamples{ HelpExampleCli("getmininginfo", "") + HelpExampleRpc("getmininginfo", "") }, }.ToString()); - + } LOCK(cs_main); UniValue obj(UniValue::VOBJ); obj.pushKV("blocks", (int)chainActive.Height()); - obj.pushKV("currentblockweight", (uint64_t)nLastBlockWeight); - obj.pushKV("currentblocktx", (uint64_t)nLastBlockTx); + if (BlockAssembler::m_last_block_weight) obj.pushKV("currentblockweight", *BlockAssembler::m_last_block_weight); + if (BlockAssembler::m_last_block_num_txs) obj.pushKV("currentblocktx", *BlockAssembler::m_last_block_num_txs); obj.pushKV("difficulty", (double)GetDifficulty(chainActive.Tip())); obj.pushKV("networkhashps", getnetworkhashps(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index ac41207404..38e2dc237e 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -27,6 +27,7 @@ #include <script/sign.h> #include <script/standard.h> #include <uint256.h> +#include <util/bip32.h> #include <util/strencodings.h> #include <validation.h> #include <validationinterface.h> @@ -858,15 +859,25 @@ UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, con RPCTypeCheckObj(prevOut, { {"redeemScript", UniValueType(UniValue::VSTR)}, - }); - UniValue v = find_value(prevOut, "redeemScript"); - if (!v.isNull()) { - std::vector<unsigned char> rsData(ParseHexV(v, "redeemScript")); + {"witnessScript", UniValueType(UniValue::VSTR)}, + }, true); + UniValue rs = find_value(prevOut, "redeemScript"); + if (!rs.isNull()) { + std::vector<unsigned char> rsData(ParseHexV(rs, "redeemScript")); CScript redeemScript(rsData.begin(), rsData.end()); keystore->AddCScript(redeemScript); // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). + // This is only for compatibility, it is encouraged to use the explicit witnessScript field instead. keystore->AddCScript(GetScriptForWitness(redeemScript)); } + UniValue ws = find_value(prevOut, "witnessScript"); + if (!ws.isNull()) { + std::vector<unsigned char> wsData(ParseHexV(ws, "witnessScript")); + CScript witnessScript(wsData.begin(), wsData.end()); + keystore->AddCScript(witnessScript); + // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). + keystore->AddCScript(GetScriptForWitness(witnessScript)); + } } } } @@ -951,7 +962,8 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, {"scriptPubKey", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "script key"}, - {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH or P2WSH) redeem script"}, + {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH) redeem script"}, + {"witnessScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2WSH or P2SH-P2WSH) witness script"}, {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount spent"}, }, }, diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 41e0f2e117..532a8028a2 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -10,6 +10,7 @@ #include <script/standard.h> #include <span.h> +#include <util/bip32.h> #include <util/system.h> #include <util/strencodings.h> @@ -25,16 +26,6 @@ namespace { typedef std::vector<uint32_t> KeyPath; -std::string FormatKeyPath(const KeyPath& path) -{ - std::string ret; - for (auto i : path) { - ret += strprintf("/%i", (i << 1) >> 1); - if (i >> 31) ret += '\''; - } - return ret; -} - /** Interface for public key objects in descriptors. */ struct PubkeyProvider { @@ -63,7 +54,7 @@ class OriginPubkeyProvider final : public PubkeyProvider std::string OriginString() const { - return HexStr(std::begin(m_origin.fingerprint), std::end(m_origin.fingerprint)) + FormatKeyPath(m_origin.path); + return HexStr(std::begin(m_origin.fingerprint), std::end(m_origin.fingerprint)) + FormatHDKeypath(m_origin.path); } public: @@ -184,7 +175,7 @@ public: } std::string ToString() const override { - std::string ret = EncodeExtPubKey(m_extkey) + FormatKeyPath(m_path); + std::string ret = EncodeExtPubKey(m_extkey) + FormatHDKeypath(m_path); if (IsRange()) { ret += "/*"; if (m_derive == DeriveType::HARDENED) ret += '\''; @@ -195,7 +186,7 @@ public: { CExtKey key; if (!GetExtKey(arg, key)) return false; - out = EncodeExtKey(key) + FormatKeyPath(m_path); + out = EncodeExtKey(key) + FormatHDKeypath(m_path); if (IsRange()) { out += "/*"; if (m_derive == DeriveType::HARDENED) out += '\''; diff --git a/src/script/sign.h b/src/script/sign.h index e229c375c7..491fb54c45 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -22,13 +22,27 @@ struct CMutableTransaction; struct KeyOriginInfo { - unsigned char fingerprint[4]; + unsigned char fingerprint[4]; //!< First 32 bits of the Hash160 of the public key at the root of the path std::vector<uint32_t> path; friend bool operator==(const KeyOriginInfo& a, const KeyOriginInfo& b) { return std::equal(std::begin(a.fingerprint), std::end(a.fingerprint), std::begin(b.fingerprint)) && a.path == b.path; } + + ADD_SERIALIZE_METHODS; + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(fingerprint); + READWRITE(path); + } + + void clear() + { + memset(fingerprint, 0, 4); + path.clear(); + } }; /** An interface to be implemented by keystores that support signing. */ diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/fuzz/deserialize.cpp index 859fba0bdc..859fba0bdc 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/fuzz/deserialize.cpp diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp new file mode 100644 index 0000000000..2c0bfa360c --- /dev/null +++ b/src/test/fuzz/script_flags.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <script/interpreter.h> +#include <script/script.h> +#include <streams.h> +#include <version.h> + +#include <test/fuzz/fuzz.h> + +/** Flags that are not forbidden by an assert */ +static bool IsValidFlagCombination(unsigned flags); + +void test_one_input(std::vector<uint8_t> buffer) +{ + CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); + try { + int nVersion; + ds >> nVersion; + ds.SetVersion(nVersion); + } catch (const std::ios_base::failure&) { + return; + } + + try { + const CTransaction tx(deserialize, ds); + const PrecomputedTransactionData txdata(tx); + + unsigned int verify_flags; + ds >> verify_flags; + + if (!IsValidFlagCombination(verify_flags)) return; + + unsigned int fuzzed_flags; + ds >> fuzzed_flags; + + for (unsigned i = 0; i < tx.vin.size(); ++i) { + CTxOut prevout; + ds >> prevout; + + const TransactionSignatureChecker checker{&tx, i, prevout.nValue, txdata}; + + ScriptError serror; + const bool ret = VerifyScript(tx.vin.at(i).scriptSig, prevout.scriptPubKey, &tx.vin.at(i).scriptWitness, verify_flags, checker, &serror); + assert(ret == (serror == SCRIPT_ERR_OK)); + + // Verify that removing flags from a passing test or adding flags to a failing test does not change the result + if (ret) { + verify_flags &= ~fuzzed_flags; + } else { + verify_flags |= fuzzed_flags; + } + if (!IsValidFlagCombination(verify_flags)) return; + + ScriptError serror_fuzzed; + const bool ret_fuzzed = VerifyScript(tx.vin.at(i).scriptSig, prevout.scriptPubKey, &tx.vin.at(i).scriptWitness, verify_flags, checker, &serror_fuzzed); + assert(ret_fuzzed == (serror_fuzzed == SCRIPT_ERR_OK)); + + assert(ret_fuzzed == ret); + } + } catch (const std::ios_base::failure&) { + return; + } +} + +static bool IsValidFlagCombination(unsigned flags) +{ + if (flags & SCRIPT_VERIFY_CLEANSTACK && ~flags & (SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS)) return false; + if (flags & SCRIPT_VERIFY_WITNESS && ~flags & SCRIPT_VERIFY_P2SH) return false; + return true; +} diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 0301901bf0..d667c26c3c 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -69,7 +69,13 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) } } - txindex.Stop(); // Stop thread before calling destructor + // shutdown sequence (c.f. Shutdown() in init.cpp) + txindex.Stop(); + + threadGroup.interrupt_all(); + threadGroup.join_all(); + + // Rest of shutdown sequence and destructors happen in ~TestingSetup() } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index c467f27836..4d04aae7e9 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -102,7 +102,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // should fail. // Capture this interaction with the upgraded_nop argument: set it when evaluating // any script flag that is implemented as an upgraded NOP code. -static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) +static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { PrecomputedTransactionData txdata(tx); // If we add many more flags, this loop can get too expensive, but we can @@ -219,11 +219,10 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) CBlock block; block = CreateAndProcessBlock({spend_tx}, p2pk_scriptPubKey); + LOCK(cs_main); BOOST_CHECK(chainActive.Tip()->GetBlockHash() == block.GetHash()); BOOST_CHECK(pcoinsTip->GetBestBlock() == block.GetHash()); - LOCK(cs_main); - // Test P2SH: construct a transaction that is valid without P2SH, and // then test validity with P2SH. { diff --git a/src/util/bip32.cpp b/src/util/bip32.cpp new file mode 100644 index 0000000000..6f176dd5ec --- /dev/null +++ b/src/util/bip32.cpp @@ -0,0 +1,66 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <sstream> +#include <stdio.h> +#include <tinyformat.h> +#include <util/bip32.h> +#include <util/strencodings.h> + + +bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath) +{ + std::stringstream ss(keypath_str); + std::string item; + bool first = true; + while (std::getline(ss, item, '/')) { + if (item.compare("m") == 0) { + if (first) { + first = false; + continue; + } + return false; + } + // Finds whether it is hardened + uint32_t path = 0; + size_t pos = item.find("'"); + if (pos != std::string::npos) { + // The hardened tick can only be in the last index of the string + if (pos != item.size() - 1) { + return false; + } + path |= 0x80000000; + item = item.substr(0, item.size() - 1); // Drop the last character which is the hardened tick + } + + // Ensure this is only numbers + if (item.find_first_not_of( "0123456789" ) != std::string::npos) { + return false; + } + uint32_t number; + if (!ParseUInt32(item, &number)) { + return false; + } + path |= number; + + keypath.push_back(path); + first = false; + } + return true; +} + +std::string FormatHDKeypath(const std::vector<uint32_t>& path) +{ + std::string ret; + for (auto i : path) { + ret += strprintf("/%i", (i << 1) >> 1); + if (i >> 31) ret += '\''; + } + return ret; +} + +std::string WriteHDKeypath(const std::vector<uint32_t>& keypath) +{ + return "m" + FormatHDKeypath(keypath); +} diff --git a/src/util/bip32.h b/src/util/bip32.h new file mode 100644 index 0000000000..7e58b79f38 --- /dev/null +++ b/src/util/bip32.h @@ -0,0 +1,19 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_BIP32_H +#define BITCOIN_UTIL_BIP32_H + +#include <attributes.h> +#include <string> +#include <vector> + +/** Parse an HD keypaths like "m/7/0'/2000". */ +NODISCARD bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath); + +/** Write HD keypaths as strings */ +std::string WriteHDKeypath(const std::vector<uint32_t>& keypath); +std::string FormatHDKeypath(const std::vector<uint32_t>& path); + +#endif // BITCOIN_UTIL_BIP32_H diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index b55547bc63..0acbb4f117 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -546,47 +546,6 @@ bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out) return true; } -bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath) -{ - std::stringstream ss(keypath_str); - std::string item; - bool first = true; - while (std::getline(ss, item, '/')) { - if (item.compare("m") == 0) { - if (first) { - first = false; - continue; - } - return false; - } - // Finds whether it is hardened - uint32_t path = 0; - size_t pos = item.find("'"); - if (pos != std::string::npos) { - // The hardened tick can only be in the last index of the string - if (pos != item.size() - 1) { - return false; - } - path |= 0x80000000; - item = item.substr(0, item.size() - 1); // Drop the last character which is the hardened tick - } - - // Ensure this is only numbers - if (item.find_first_not_of( "0123456789" ) != std::string::npos) { - return false; - } - uint32_t number; - if (!ParseUInt32(item, &number)) { - return false; - } - path |= number; - - keypath.push_back(path); - first = false; - } - return true; -} - void Downcase(std::string& str) { std::transform(str.begin(), str.end(), str.begin(), [](char c){return ToLower(c);}); diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 59eefff566..cf77044094 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -197,9 +197,6 @@ bool ConvertBits(const O& outfn, I it, I end) { return true; } -/** Parse an HD keypaths like "m/7/0'/2000". */ -NODISCARD bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath); - /** * Converts the given character to its lowercase equivalent. * This function is locale independent. It only converts uppercase diff --git a/src/util/system.h b/src/util/system.h index 54d4cf2e58..8867e49478 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -16,6 +16,7 @@ #include <attributes.h> #include <compat.h> +#include <compat/assumptions.h> #include <fs.h> #include <logging.h> #include <sync.h> diff --git a/src/validation.h b/src/validation.h index 49f73e4c9b..1975846b69 100644 --- a/src/validation.h +++ b/src/validation.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,8 +14,8 @@ #include <coins.h> #include <crypto/common.h> // for ReadLE64 #include <fs.h> -#include <protocol.h> // For CMessageHeader::MessageStartChars #include <policy/feerate.h> +#include <protocol.h> // For CMessageHeader::MessageStartChars #include <script/script_error.h> #include <sync.h> #include <versionbits.h> @@ -152,8 +152,6 @@ extern CTxMemPool mempool; extern std::atomic_bool g_is_mempool_loaded; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern BlockMap& mapBlockIndex GUARDED_BY(cs_main); -extern uint64_t nLastBlockTx; -extern uint64_t nLastBlockWeight; extern const std::string strMessageMagic; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index ec49efcf22..930274f8a1 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -13,6 +13,7 @@ #include <script/script.h> #include <script/standard.h> #include <sync.h> +#include <util/bip32.h> #include <util/system.h> #include <util/time.h> #include <validation.h> @@ -850,7 +851,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) } else { file << "change=1"; } - file << strprintf(" # addr=%s%s\n", strAddr, (pwallet->mapKeyMetadata[keyid].hdKeypath.size() > 0 ? " hdkeypath="+pwallet->mapKeyMetadata[keyid].hdKeypath : "")); + file << strprintf(" # addr=%s%s\n", strAddr, (pwallet->mapKeyMetadata[keyid].has_key_origin ? " hdkeypath="+WriteHDKeypath(pwallet->mapKeyMetadata[keyid].key_origin.path) : "")); } } file << "\n"; @@ -887,6 +888,7 @@ struct ImportData // Output data std::set<CScript> import_scripts; std::map<CKeyID, bool> used_keys; //!< Import these private keys if available (the value indicates whether if the key is required for solvability) + std::map<CKeyID, KeyOriginInfo> key_origins; }; enum class ScriptContext @@ -965,7 +967,7 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d } } -static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data) +static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data, std::vector<CKeyID>& ordered_pubkeys) { UniValue warnings(UniValue::VARR); @@ -1036,6 +1038,7 @@ static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CP throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" is not a valid public key"); } pubkey_map.emplace(pubkey.GetID(), pubkey); + ordered_pubkeys.push_back(pubkey.GetID()); } for (size_t i = 0; i < keys.size(); ++i) { const auto& str = keys[i].get_str(); @@ -1108,7 +1111,7 @@ static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CP return warnings; } -static UniValue ProcessImportDescriptor(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data) +static UniValue ProcessImportDescriptor(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data, std::vector<CKeyID>& ordered_pubkeys) { UniValue warnings(UniValue::VARR); @@ -1142,21 +1145,24 @@ static UniValue ProcessImportDescriptor(ImportData& import_data, std::map<CKeyID const UniValue& priv_keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); - FlatSigningProvider out_keys; - // Expand all descriptors to get public keys and scripts. // TODO: get private keys from descriptors too for (int i = range_start; i <= range_end; ++i) { + FlatSigningProvider out_keys; std::vector<CScript> scripts_temp; parsed_desc->Expand(i, keys, scripts_temp, out_keys); std::copy(scripts_temp.begin(), scripts_temp.end(), std::inserter(script_pub_keys, script_pub_keys.end())); - } + for (const auto& key_pair : out_keys.pubkeys) { + ordered_pubkeys.push_back(key_pair.first); + } - for (const auto& x : out_keys.scripts) { - import_data.import_scripts.emplace(x.second); - } + for (const auto& x : out_keys.scripts) { + import_data.import_scripts.emplace(x.second); + } - std::copy(out_keys.pubkeys.begin(), out_keys.pubkeys.end(), std::inserter(pubkey_map, pubkey_map.end())); + std::copy(out_keys.pubkeys.begin(), out_keys.pubkeys.end(), std::inserter(pubkey_map, pubkey_map.end())); + import_data.key_origins.insert(out_keys.origins.begin(), out_keys.origins.end()); + } for (size_t i = 0; i < priv_keys.size(); ++i) { const auto& str = priv_keys[i].get_str(); @@ -1205,19 +1211,26 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal addresses should not have a label"); } const std::string& label = data.exists("label") ? data["label"].get_str() : ""; + const bool add_keypool = data.exists("keypool") ? data["keypool"].get_bool() : false; + + // Add to keypool only works with privkeys disabled + if (add_keypool && !pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Keys can only be imported to the keypool when private keys are disabled"); + } ImportData import_data; std::map<CKeyID, CPubKey> pubkey_map; std::map<CKeyID, CKey> privkey_map; std::set<CScript> script_pub_keys; + std::vector<CKeyID> ordered_pubkeys; bool have_solving_data; if (data.exists("scriptPubKey") && data.exists("desc")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Both a descriptor and a scriptPubKey should not be provided."); } else if (data.exists("scriptPubKey")) { - warnings = ProcessImportLegacy(import_data, pubkey_map, privkey_map, script_pub_keys, have_solving_data, data); + warnings = ProcessImportLegacy(import_data, pubkey_map, privkey_map, script_pub_keys, have_solving_data, data, ordered_pubkeys); } else if (data.exists("desc")) { - warnings = ProcessImportDescriptor(import_data, pubkey_map, privkey_map, script_pub_keys, have_solving_data, data); + warnings = ProcessImportDescriptor(import_data, pubkey_map, privkey_map, script_pub_keys, have_solving_data, data, ordered_pubkeys); } else { throw JSONRPCError(RPC_INVALID_PARAMETER, "Either a descriptor or scriptPubKey must be provided."); } @@ -1239,27 +1252,40 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con for (const auto& entry : import_data.import_scripts) { if (!pwallet->HaveCScript(CScriptID(entry)) && !pwallet->AddCScript(entry)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding script to wallet"); + } + } + for (const auto& entry : privkey_map) { + const CKey& key = entry.second; + CPubKey pubkey = key.GetPubKey(); + const CKeyID& id = entry.first; + assert(key.VerifyPubKey(pubkey)); + pwallet->mapKeyMetadata[id].nCreateTime = timestamp; + // If the private key is not present in the wallet, insert it. + if (!pwallet->HaveKey(id) && !pwallet->AddKeyPubKey(key, pubkey)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); + } + pwallet->UpdateTimeFirstKey(timestamp); + } + for (const CKeyID& id : ordered_pubkeys) { + auto entry = pubkey_map.find(id); + if (entry == pubkey_map.end()) { + continue; } - } - for (const auto& entry : privkey_map) { - const CKey& key = entry.second; - CPubKey pubkey = key.GetPubKey(); - const CKeyID& id = entry.first; - assert(key.VerifyPubKey(pubkey)); - pwallet->mapKeyMetadata[id].nCreateTime = timestamp; - // If the private key is not present in the wallet, insert it. - if (!pwallet->HaveKey(id) && !pwallet->AddKeyPubKey(key, pubkey)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); - } - pwallet->UpdateTimeFirstKey(timestamp); - } - for (const auto& entry : pubkey_map) { - const CPubKey& pubkey = entry.second; - const CKeyID& id = entry.first; - CPubKey temp; - if (!pwallet->GetPubKey(id, temp) && !pwallet->AddWatchOnly(GetScriptForRawPubKey(pubkey), timestamp)) { + const CPubKey& pubkey = entry->second; + CPubKey temp; + if (!pwallet->GetPubKey(id, temp) && !pwallet->AddWatchOnly(GetScriptForRawPubKey(pubkey), timestamp)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); } + const auto& key_orig_it = import_data.key_origins.find(id); + if (key_orig_it != import_data.key_origins.end()) { + pwallet->AddKeyOrigin(pubkey, key_orig_it->second); + } + pwallet->mapKeyMetadata[id].nCreateTime = timestamp; + + // Add to keypool only works with pubkeys + if (add_keypool) { + pwallet->AddKeypoolPubkey(pubkey, internal); + } } for (const CScript& script : script_pub_keys) { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 9e362cf167..22ea592738 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -27,6 +27,7 @@ #include <script/sign.h> #include <shutdown.h> #include <timedata.h> +#include <util/bip32.h> #include <util/system.h> #include <util/moneystr.h> #include <wallet/coincontrol.h> @@ -172,18 +173,12 @@ static UniValue getnewaddress(const JSONRPCRequest& request) }, }.ToString()); - // Belt and suspenders check for disabled private keys - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); - } - LOCK(pwallet->cs_wallet); if (!pwallet->CanGetAddresses()) { throw JSONRPCError(RPC_WALLET_ERROR, "Error: This wallet has no available keys"); } - // Parse the label first so we don't generate a key if there's an error std::string label; if (!request.params[0].isNull()) @@ -239,11 +234,6 @@ static UniValue getrawchangeaddress(const JSONRPCRequest& request) }, }.ToString()); - // Belt and suspenders check for disabled private keys - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); - } - LOCK(pwallet->cs_wallet); if (!pwallet->CanGetAddresses(true)) { @@ -2418,7 +2408,6 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" " \"hdseedid\": \"<hash160>\" (string, optional) the Hash160 of the HD seed (only present when HD is enabled)\n" - " \"hdmasterkeyid\": \"<hash160>\" (string, optional) alias for hdseedid retained for backwards-compatibility. Will be removed in V0.18.\n" " \"private_keys_enabled\": true|false (boolean) false if privatekeys are disabled for this wallet (enforced watch-only wallet)\n" "}\n" }, @@ -2447,7 +2436,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) obj.pushKV("keypoololdest", pwallet->GetOldestKeyPoolTime()); obj.pushKV("keypoolsize", (int64_t)kpExternalSize); CKeyID seed_id = pwallet->GetHDChain().seed_id; - if (!seed_id.IsNull() && pwallet->CanSupportFeature(FEATURE_HD_SPLIT)) { + if (pwallet->CanSupportFeature(FEATURE_HD_SPLIT)) { obj.pushKV("keypoolsize_hd_internal", (int64_t)(pwallet->GetKeyPoolSize() - kpExternalSize)); } if (pwallet->IsCrypted()) { @@ -2456,7 +2445,6 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) obj.pushKV("paytxfee", ValueFromAmount(pwallet->m_pay_tx_fee.GetFeePerK())); if (!seed_id.IsNull()) { obj.pushKV("hdseedid", seed_id.GetHex()); - obj.pushKV("hdmasterkeyid", seed_id.GetHex()); } obj.pushKV("private_keys_enabled", !pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); return obj; @@ -2770,7 +2758,8 @@ static UniValue listunspent(const JSONRPCRequest& request) " \"scriptPubKey\" : \"key\", (string) the script key\n" " \"amount\" : x.xxx, (numeric) the transaction output amount in " + CURRENCY_UNIT + "\n" " \"confirmations\" : n, (numeric) The number of confirmations\n" - " \"redeemScript\" : n (string) The redeemScript if scriptPubKey is P2SH\n" + " \"redeemScript\" : \"script\" (string) The redeemScript if scriptPubKey is P2SH" + " \"witnessScript\" : \"script\" (string) witnessScript if the scriptPubKey is P2WSH or P2SH-P2WSH\n" " \"spendable\" : xxx, (bool) Whether we have the private keys to spend this output\n" " \"solvable\" : xxx, (bool) Whether we know how to spend this output, ignoring the lack of keys\n" " \"desc\" : xxx, (string, only when solvable) A descriptor for spending this output\n" @@ -2884,6 +2873,28 @@ static UniValue listunspent(const JSONRPCRequest& request) CScript redeemScript; if (pwallet->GetCScript(hash, redeemScript)) { entry.pushKV("redeemScript", HexStr(redeemScript.begin(), redeemScript.end())); + // Now check if the redeemScript is actually a P2WSH script + CTxDestination witness_destination; + if (redeemScript.IsPayToWitnessScriptHash()) { + bool extracted = ExtractDestination(redeemScript, witness_destination); + assert(extracted); + // Also return the witness script + const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(witness_destination); + CScriptID id; + CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); + CScript witnessScript; + if (pwallet->GetCScript(id, witnessScript)) { + entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); + } + } + } + } else if (scriptPubKey.IsPayToWitnessScriptHash()) { + const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(address); + CScriptID id; + CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); + CScript witnessScript; + if (pwallet->GetCScript(id, witnessScript)) { + entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); } } } @@ -3139,7 +3150,8 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, {"scriptPubKey", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "script key"}, - {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH or P2WSH) redeem script"}, + {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH) redeem script"}, + {"witnessScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2WSH or P2SH-P2WSH) witness script"}, {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount spent"}, }, }, @@ -3660,7 +3672,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) " \"timestamp\" : timestamp, (number, optional) The creation time of the key if available in seconds since epoch (Jan 1 1970 GMT)\n" " \"hdkeypath\" : \"keypath\" (string, optional) The HD keypath if the key is HD and available\n" " \"hdseedid\" : \"<hash160>\" (string, optional) The Hash160 of the HD seed\n" - " \"hdmasterkeyid\" : \"<hash160>\" (string, optional) alias for hdseedid maintained for backwards compatibility. Will be removed in V0.18.\n" + " \"hdmasterfingerprint\" : \"<hash160>\" (string, optional) The fingperint of the master key.\n" " \"labels\" (object) Array of labels associated with the address.\n" " [\n" " { (json object of label data)\n" @@ -3723,10 +3735,10 @@ UniValue getaddressinfo(const JSONRPCRequest& request) } if (meta) { ret.pushKV("timestamp", meta->nCreateTime); - if (!meta->hdKeypath.empty()) { - ret.pushKV("hdkeypath", meta->hdKeypath); + if (meta->has_key_origin) { + ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); - ret.pushKV("hdmasterkeyid", meta->hd_seed_id.GetHex()); + ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint, meta->key_origin.fingerprint + 4)); } } diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index e89d4121bc..2a3149de46 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -4,6 +4,7 @@ #include <key_io.h> #include <script/sign.h> +#include <util/bip32.h> #include <util/strencodings.h> #include <wallet/psbtwallet.h> #include <wallet/rpcwallet.h> diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 063015d1d8..d174e308f0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -27,6 +27,7 @@ #include <shutdown.h> #include <timedata.h> #include <txmempool.h> +#include <util/bip32.h> #include <util/moneystr.h> #include <wallet/fees.h> @@ -255,16 +256,25 @@ void CWallet::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey if (internal) { chainChildKey.Derive(childKey, hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); metadata.hdKeypath = "m/0'/1'/" + std::to_string(hdChain.nInternalChainCounter) + "'"; + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(1 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); hdChain.nInternalChainCounter++; } else { chainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); hdChain.nExternalChainCounter++; } } while (HaveKey(childKey.key.GetPubKey().GetID())); secret = childKey.key; metadata.hd_seed_id = hdChain.seed_id; + CKeyID master_id = masterKey.key.GetPubKey().GetID(); + std::copy(master_id.begin(), master_id.begin() + 4, metadata.key_origin.fingerprint); + metadata.has_key_origin = true; // update the chain model in the database if (!batch.WriteHDChain(hdChain)) throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); @@ -348,6 +358,47 @@ void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata m_script_metadata[script_id] = meta; } +// Writes a keymetadata for a public key. overwrite specifies whether to overwrite an existing metadata for that key if there exists one. +bool CWallet::WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, const bool overwrite) +{ + return WalletBatch(*database).WriteKeyMetadata(meta, pubkey, overwrite); +} + +void CWallet::UpgradeKeyMetadata() +{ + AssertLockHeld(cs_wallet); // mapKeyMetadata + if (IsLocked() || IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { + return; + } + + for (auto& meta_pair : mapKeyMetadata) { + CKeyMetadata& meta = meta_pair.second; + if (!meta.hd_seed_id.IsNull() && !meta.has_key_origin && meta.hdKeypath != "s") { // If the hdKeypath is "s", that's the seed and it doesn't have a key origin + CKey key; + GetKey(meta.hd_seed_id, key); + CExtKey masterKey; + masterKey.SetSeed(key.begin(), key.size()); + // Add to map + CKeyID master_id = masterKey.key.GetPubKey().GetID(); + std::copy(master_id.begin(), master_id.begin() + 4, meta.key_origin.fingerprint); + if (!ParseHDKeypath(meta.hdKeypath, meta.key_origin.path)) { + throw std::runtime_error("Invalid stored hdKeypath"); + } + meta.has_key_origin = true; + if (meta.nVersion < CKeyMetadata::VERSION_WITH_KEY_ORIGIN) { + meta.nVersion = CKeyMetadata::VERSION_WITH_KEY_ORIGIN; + } + + // Write meta to wallet + CPubKey pubkey; + if (GetPubKey(meta_pair.first, pubkey)) { + WriteKeyMetadata(meta, pubkey, true); + } + } + } + SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); +} + bool CWallet::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) { return CCryptoKeyStore::AddCryptedKey(vchPubKey, vchCryptedSecret); @@ -446,8 +497,11 @@ bool CWallet::Unlock(const SecureString& strWalletPassphrase, bool accept_no_key return false; if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, _vMasterKey)) continue; // try another master key - if (CCryptoKeyStore::Unlock(_vMasterKey, accept_no_keys)) + if (CCryptoKeyStore::Unlock(_vMasterKey, accept_no_keys)) { + // Now that we've unlocked, upgrade the key metadata + UpgradeKeyMetadata(); return true; + } } } return false; @@ -1407,6 +1461,7 @@ CPubKey CWallet::DeriveNewSeed(const CKey& key) // set the hd keypath to "s" -> Seed, refers the seed to itself metadata.hdKeypath = "s"; + metadata.has_key_origin = false; metadata.hd_seed_id = seed.GetID(); { @@ -2778,8 +2833,8 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // post-backup change. // Reserve a new key pair from key pool - if (IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - strFailReason = _("Can't generate a change-address key. Private keys are disabled for this wallet."); + if (!CanGetAddresses(true)) { + strFailReason = _("Can't generate a change-address key. No keys in the internal keypool and can't generate any keys."); return false; } CPubKey vchPubKey; @@ -3388,20 +3443,8 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) internal = true; } - assert(m_max_keypool_index < std::numeric_limits<int64_t>::max()); // How in the hell did you use so many keys? - int64_t index = ++m_max_keypool_index; - CPubKey pubkey(GenerateNewKey(batch, internal)); - if (!batch.WritePool(index, CKeyPool(pubkey, internal))) { - throw std::runtime_error(std::string(__func__) + ": writing generated key failed"); - } - - if (internal) { - setInternalKeyPool.insert(index); - } else { - setExternalKeyPool.insert(index); - } - m_pool_key_to_index[pubkey.GetID()] = index; + AddKeypoolPubkeyWithDB(pubkey, internal, batch); } if (missingInternal + missingExternal > 0) { WalletLogPrintf("keypool added %d keys (%d internal), size=%u (%u internal)\n", missingInternal + missingExternal, missingInternal, setInternalKeyPool.size() + setExternalKeyPool.size() + set_pre_split_keypool.size(), setInternalKeyPool.size()); @@ -3411,6 +3454,29 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) return true; } +void CWallet::AddKeypoolPubkey(const CPubKey& pubkey, const bool internal) +{ + WalletBatch batch(*database); + AddKeypoolPubkeyWithDB(pubkey, internal, batch); + NotifyCanGetAddressesChanged(); +} + +void CWallet::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch) +{ + LOCK(cs_wallet); + assert(m_max_keypool_index < std::numeric_limits<int64_t>::max()); // How in the hell did you use so many keys? + int64_t index = ++m_max_keypool_index; + if (!batch.WritePool(index, CKeyPool(pubkey, internal))) { + throw std::runtime_error(std::string(__func__) + ": writing imported pubkey failed"); + } + if (internal) { + setInternalKeyPool.insert(index); + } else { + setExternalKeyPool.insert(index); + } + m_pool_key_to_index[pubkey.GetID()] = index; +} + bool CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal) { nIndex = -1; @@ -3421,7 +3487,8 @@ bool CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRe if (!IsLocked()) TopUpKeyPool(); - bool fReturningInternal = IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT) && fRequestedInternal; + bool fReturningInternal = fRequestedInternal; + fReturningInternal &= (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) || IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); bool use_split_keypool = set_pre_split_keypool.empty(); std::set<int64_t>& setKeyPool = use_split_keypool ? (fReturningInternal ? setInternalKeyPool : setExternalKeyPool) : set_pre_split_keypool; @@ -3438,7 +3505,8 @@ bool CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRe if (!batch.ReadPool(nIndex, keypool)) { throw std::runtime_error(std::string(__func__) + ": read failed"); } - if (!HaveKey(keypool.vchPubKey.GetID())) { + CPubKey pk; + if (!GetPubKey(keypool.vchPubKey.GetID(), pk)) { throw std::runtime_error(std::string(__func__) + ": unknown key in key pool"); } // If the key was pre-split keypool, we don't care about what type it is @@ -3492,7 +3560,7 @@ bool CWallet::GetKeyFromPool(CPubKey& result, bool internal) { LOCK(cs_wallet); int64_t nIndex; - if (!ReserveKeyFromKeyPool(nIndex, keypool, internal)) { + if (!ReserveKeyFromKeyPool(nIndex, keypool, internal) && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { if (IsLocked()) return false; WalletBatch batch(*database); result = GenerateNewKey(batch, internal); @@ -4487,18 +4555,21 @@ bool CWallet::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& info) const meta = it->second; } } - if (!meta.hdKeypath.empty()) { - if (!ParseHDKeypath(meta.hdKeypath, info.path)) return false; - // Get the proper master key id - CKey key; - GetKey(meta.hd_seed_id, key); - CExtKey masterKey; - masterKey.SetSeed(key.begin(), key.size()); - // Compute identifier - CKeyID masterid = masterKey.key.GetPubKey().GetID(); - std::copy(masterid.begin(), masterid.begin() + 4, info.fingerprint); + if (meta.has_key_origin) { + std::copy(meta.key_origin.fingerprint, meta.key_origin.fingerprint + 4, info.fingerprint); + info.path = meta.key_origin.path; } else { // Single pubkeys get the master fingerprint of themselves std::copy(keyID.begin(), keyID.begin() + 4, info.fingerprint); } return true; } + +bool CWallet::AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info) +{ + LOCK(cs_wallet); + std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); + mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; + mapKeyMetadata[pubkey.GetID()].has_key_origin = true; + mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path); + return WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true); +} diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 5846ac0f3e..2a5d6caaf8 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -135,6 +135,9 @@ enum WalletFlags : uint64_t { // wallet flags in the upper section (> 1 << 31) will lead to not opening the wallet if flag is unknown // unknown wallet flags in the lower section <= (1 << 31) will be tolerated + // Indicates that the metadata has already been upgraded to contain key origins + WALLET_FLAG_KEY_ORIGIN_METADATA = (1ULL << 1), + // will enforce the rule that the wallet can't contain any private keys (only watch-only/pubkeys) WALLET_FLAG_DISABLE_PRIVATE_KEYS = (1ULL << 32), @@ -151,7 +154,7 @@ enum WalletFlags : uint64_t { WALLET_FLAG_BLANK_WALLET = (1ULL << 33), }; -static constexpr uint64_t g_known_wallet_flags = WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET; +static constexpr uint64_t g_known_wallet_flags = WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET | WALLET_FLAG_KEY_ORIGIN_METADATA; /** A key pool entry */ class CKeyPool @@ -776,6 +779,8 @@ public: // Map from Script ID to key metadata (for watch-only keys). std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_wallet); + bool WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, bool overwrite); + typedef std::map<unsigned int, CMasterKey> MasterKeyMap; MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID = 0; @@ -866,6 +871,8 @@ public: //! Load metadata (used by LoadWallet) void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo + void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; nWalletMaxVersion = std::max(nWalletMaxVersion, nVersion); return true; } void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -994,6 +1001,8 @@ public: bool NewKeyPool(); size_t KeypoolCountExternalKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool TopUpKeyPool(unsigned int kpSize = 0); + void AddKeypoolPubkey(const CPubKey& pubkey, const bool internal); + void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); /** * Reserves a key from the keypool and sets nIndex to its index @@ -1212,6 +1221,9 @@ public: /** Implement lookup of key origin information through wallet key metadata. */ bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; + + /** Add a KeyOriginInfo to the wallet */ + bool AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info); }; /** A key allocated from the key pool. */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 6e037808e3..2783f83fd6 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -57,9 +57,14 @@ bool WalletBatch::EraseTx(uint256 hash) return EraseIC(std::make_pair(std::string("tx"), hash)); } +bool WalletBatch::WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, const bool overwrite) +{ + return WriteIC(std::make_pair(std::string("keymeta"), pubkey), meta, overwrite); +} + bool WalletBatch::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, const CKeyMetadata& keyMeta) { - if (!WriteIC(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta, false)) { + if (!WriteKeyMetadata(keyMeta, vchPubKey, false)) { return false; } @@ -76,7 +81,7 @@ bool WalletBatch::WriteCryptedKey(const CPubKey& vchPubKey, const std::vector<unsigned char>& vchCryptedSecret, const CKeyMetadata &keyMeta) { - if (!WriteIC(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta)) { + if (!WriteKeyMetadata(keyMeta, vchPubKey, true)) { return false; } @@ -529,6 +534,14 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (wss.fAnyUnordered) result = pwallet->ReorderTransactions(); + // Upgrade all of the wallet keymetadata to have the hd master key id + // This operation is not atomic, but if it fails, updated entries are still backwards compatible with older software + try { + pwallet->UpgradeKeyMetadata(); + } catch (...) { + result = DBErrors::CORRUPT; + } + return result; } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 5584407a56..0532a55ff5 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -8,6 +8,7 @@ #include <amount.h> #include <primitives/transaction.h> +#include <script/sign.h> #include <wallet/db.h> #include <key.h> @@ -93,11 +94,14 @@ class CKeyMetadata public: static const int VERSION_BASIC=1; static const int VERSION_WITH_HDDATA=10; - static const int CURRENT_VERSION=VERSION_WITH_HDDATA; + static const int VERSION_WITH_KEY_ORIGIN = 12; + static const int CURRENT_VERSION=VERSION_WITH_KEY_ORIGIN; int nVersion; int64_t nCreateTime; // 0 means unknown - std::string hdKeypath; //optional HD/bip32 keypath + std::string hdKeypath; //optional HD/bip32 keypath. Still used to determine whether a key is a seed. Also kept for backwards compatibility CKeyID hd_seed_id; //id of the HD seed used to derive this key + KeyOriginInfo key_origin; // Key origin info with path and fingerprint + bool has_key_origin = false; //< Whether the key_origin is useful CKeyMetadata() { @@ -120,6 +124,11 @@ public: READWRITE(hdKeypath); READWRITE(hd_seed_id); } + if (this->nVersion >= VERSION_WITH_KEY_ORIGIN) + { + READWRITE(key_origin); + READWRITE(has_key_origin); + } } void SetNull() @@ -128,6 +137,8 @@ public: nCreateTime = 0; hdKeypath.clear(); hd_seed_id.SetNull(); + key_origin.clear(); + has_key_origin = false; } }; @@ -177,6 +188,7 @@ public: bool WriteTx(const CWalletTx& wtx); bool EraseTx(uint256 hash); + bool WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, const bool overwrite); bool WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, const CKeyMetadata &keyMeta); bool WriteCryptedKey(const CPubKey& vchPubKey, const std::vector<unsigned char>& vchCryptedSecret, const CKeyMetadata &keyMeta); bool WriteMasterKey(unsigned int nID, const CMasterKey& kMasterKey); diff --git a/test/config.ini.in b/test/config.ini.in index 28abee2a3d..6b7ef70659 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -16,4 +16,5 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @ENABLE_WALLET_TRUE@ENABLE_WALLET=true @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true +@ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 5dafb11ac5..df8fe23a2e 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mining RPCs @@ -11,7 +11,10 @@ import copy from decimal import Decimal -from test_framework.blocktools import create_coinbase +from test_framework.blocktools import ( + create_coinbase, + TIME_GENESIS_BLOCK, +) from test_framework.messages import ( CBlock, CBlockHeader, @@ -25,9 +28,11 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, bytes_to_hex_str as b2x, + connect_nodes_bi, ) from test_framework.script import CScriptNum + def assert_template(node, block, expect, rehash=True): if rehash: block.hashMerkleRoot = block.calc_merkle_root() @@ -38,9 +43,22 @@ def assert_template(node, block, expect, rehash=True): class MiningTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.setup_clean_chain = False + self.setup_clean_chain = True + + def mine_chain(self): + self.log.info('Create some old blocks') + for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): + self.nodes[0].setmocktime(t) + self.nodes[0].generate(1) + mining_info = self.nodes[0].getmininginfo() + assert_equal(mining_info['blocks'], 200) + assert_equal(mining_info['currentblocktx'], 0) + assert_equal(mining_info['currentblockweight'], 4000) + self.restart_node(0) + connect_nodes_bi(self.nodes, 0, 1) def run_test(self): + self.mine_chain() node = self.nodes[0] def assert_submitblock(block, result_str_1, result_str_2=None): @@ -53,8 +71,8 @@ class MiningTest(BitcoinTestFramework): mining_info = node.getmininginfo() assert_equal(mining_info['blocks'], 200) assert_equal(mining_info['chain'], 'regtest') - assert_equal(mining_info['currentblocktx'], 0) - assert_equal(mining_info['currentblockweight'], 0) + assert 'currentblocktx' not in mining_info + assert 'currentblockweight' not in mining_info assert_equal(mining_info['difficulty'], Decimal('4.656542373906925E-10')) assert_equal(mining_info['networkhashps'], Decimal('0.003333333333333334')) assert_equal(mining_info['pooledtx'], 0) diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index 291538df64..56e2c73a90 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -5,14 +5,17 @@ """Test transaction signing using the signrawtransaction* RPCs.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, hex_str_to_bytes +from test_framework.messages import sha256 +from test_framework.script import CScript, OP_0 +from decimal import Decimal class SignRawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 1 - self.extra_args = [["-deprecatedrpc=signrawtransaction"]] + self.num_nodes = 2 + self.extra_args = [["-deprecatedrpc=signrawtransaction"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -143,9 +146,33 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert_equal(rawTxSigned['errors'][1]['witness'], ["304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee01", "025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee6357"]) assert not rawTxSigned['errors'][0]['witness'] + def witness_script_test(self): + # Now test signing transaction to P2SH-P2WSH addresses without wallet + # Create a new P2SH-P2WSH 1-of-1 multisig address: + embedded_address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress()) + embedded_privkey = self.nodes[1].dumpprivkey(embedded_address["address"]) + p2sh_p2wsh_address = self.nodes[1].addmultisigaddress(1, [embedded_address["pubkey"]], "", "p2sh-segwit") + # send transaction to P2SH-P2WSH 1-of-1 multisig address + self.nodes[0].generate(101) + self.nodes[0].sendtoaddress(p2sh_p2wsh_address["address"], 49.999) + self.nodes[0].generate(1) + self.sync_all() + # Find the UTXO for the transaction node[1] should have received, check witnessScript matches + unspent_output = self.nodes[1].listunspent(0, 999999, [p2sh_p2wsh_address["address"]])[0] + assert_equal(unspent_output["witnessScript"], p2sh_p2wsh_address["redeemScript"]) + p2sh_redeemScript = CScript([OP_0, sha256(hex_str_to_bytes(p2sh_p2wsh_address["redeemScript"]))]) + assert_equal(unspent_output["redeemScript"], bytes_to_hex_str(p2sh_redeemScript)) + # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys + spending_tx = self.nodes[0].createrawtransaction([unspent_output], {self.nodes[1].getnewaddress(): Decimal("49.998")}) + spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [unspent_output]) + # Check the signing completed successfully + assert 'complete' in spending_tx_signed + assert_equal(spending_tx_signed['complete'], True) + def run_test(self): self.successful_signing_test() self.script_verification_error_test() + self.witness_script_test() self.test_with_lock_outputs() diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 6b47cae4c3..15f4502994 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" @@ -43,9 +43,13 @@ from io import BytesIO MAX_BLOCK_SIGOPS = 20000 +# Genesis block time (regtest) +TIME_GENESIS_BLOCK = 1296688602 + # From BIP141 WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed" + def create_block(hashprev, coinbase, ntime=None, *, version=1): """Create a block (with regtest difficulty).""" block = CBlock() diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index 9fd2650d78..7ec72b8649 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -31,8 +31,8 @@ class CreateWalletTest(BitcoinTestFramework): self.log.info("Test disableprivatekeys creation.") self.nodes[0].createwallet(wallet_name='w1', disable_private_keys=True) w1 = node.get_wallet_rpc('w1') - assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w1.getnewaddress) - assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w1.getrawchangeaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w1.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w1.getrawchangeaddress) w1.importpubkey(w0.getaddressinfo(address1)['pubkey']) self.log.info('Test that private keys cannot be imported') @@ -48,8 +48,8 @@ class CreateWalletTest(BitcoinTestFramework): self.log.info("Test blank creation with private keys disabled.") self.nodes[0].createwallet(wallet_name='w2', disable_private_keys=True, blank=True) w2 = node.get_wallet_rpc('w2') - assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w2.getnewaddress) - assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w2.getrawchangeaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w2.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w2.getrawchangeaddress) w2.importpubkey(w0.getaddressinfo(address1)['pubkey']) self.log.info("Test blank creation with private keys enabled.") @@ -89,12 +89,12 @@ class CreateWalletTest(BitcoinTestFramework): self.nodes[0].createwallet(wallet_name='w5', disable_private_keys=True, blank=True) w5 = node.get_wallet_rpc('w5') assert_equal(w5.getwalletinfo()['keypoolsize'], 0) - assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getnewaddress) - assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getrawchangeaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getrawchangeaddress) # Encrypt the wallet w5.encryptwallet('pass') - assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getnewaddress) - assert_raises_rpc_error(-4, "Error: Private keys are disabled for this wallet", w5.getrawchangeaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getrawchangeaddress) if __name__ == '__main__': CreateWalletTest().main() diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index eb42531693..61245e5104 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -27,7 +27,6 @@ class WalletHDTest(BitcoinTestFramework): def run_test(self): # Make sure we use hd, keep masterkeyid masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - assert_equal(masterkeyid, self.nodes[1].getwalletinfo()['hdmasterkeyid']) assert_equal(len(masterkeyid), 40) # create an internal key @@ -53,7 +52,6 @@ class WalletHDTest(BitcoinTestFramework): hd_info = self.nodes[1].getaddressinfo(hd_add) assert_equal(hd_info["hdkeypath"], "m/0'/0'/"+str(i)+"'") assert_equal(hd_info["hdseedid"], masterkeyid) - assert_equal(hd_info["hdmasterkeyid"], masterkeyid) self.nodes[0].sendtoaddress(hd_add, 1) self.nodes[0].generate(1) self.nodes[0].sendtoaddress(non_hd_add, 1) @@ -83,7 +81,6 @@ class WalletHDTest(BitcoinTestFramework): hd_info_2 = self.nodes[1].getaddressinfo(hd_add_2) assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(i)+"'") assert_equal(hd_info_2["hdseedid"], masterkeyid) - assert_equal(hd_info_2["hdmasterkeyid"], masterkeyid) assert_equal(hd_add, hd_add_2) connect_nodes_bi(self.nodes, 0, 1) self.sync_all() diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index 7cce72b39f..46e3ab77c8 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -625,6 +625,182 @@ class ImportMultiTest(BitcoinTestFramework): ismine=False, iswatchonly=False) + # Import pubkeys with key origin info + self.log.info("Addresses should have hd keypath and master key id after import with key origin") + pub_addr = self.nodes[1].getnewaddress() + pub_addr = self.nodes[1].getnewaddress() + info = self.nodes[1].getaddressinfo(pub_addr) + pub = info['pubkey'] + pub_keypath = info['hdkeypath'] + pub_fpr = info['hdmasterfingerprint'] + result = self.nodes[0].importmulti( + [{ + 'desc' : "wpkh([" + pub_fpr + pub_keypath[1:] +"]" + pub + ")", + "timestamp": "now", + }] + ) + assert result[0]['success'] + pub_import_info = self.nodes[0].getaddressinfo(pub_addr) + assert_equal(pub_import_info['hdmasterfingerprint'], pub_fpr) + assert_equal(pub_import_info['pubkey'], pub) + assert_equal(pub_import_info['hdkeypath'], pub_keypath) + + # Import privkeys with key origin info + priv_addr = self.nodes[1].getnewaddress() + info = self.nodes[1].getaddressinfo(priv_addr) + priv = self.nodes[1].dumpprivkey(priv_addr) + priv_keypath = info['hdkeypath'] + priv_fpr = info['hdmasterfingerprint'] + result = self.nodes[0].importmulti( + [{ + 'desc' : "wpkh([" + priv_fpr + priv_keypath[1:] + "]" + priv + ")", + "timestamp": "now", + }] + ) + assert result[0]['success'] + priv_import_info = self.nodes[0].getaddressinfo(priv_addr) + assert_equal(priv_import_info['hdmasterfingerprint'], priv_fpr) + assert_equal(priv_import_info['hdkeypath'], priv_keypath) + + # Make sure the key origin info are still there after a restart + self.stop_nodes() + self.start_nodes() + import_info = self.nodes[0].getaddressinfo(pub_addr) + assert_equal(import_info['hdmasterfingerprint'], pub_fpr) + assert_equal(import_info['hdkeypath'], pub_keypath) + import_info = self.nodes[0].getaddressinfo(priv_addr) + assert_equal(import_info['hdmasterfingerprint'], priv_fpr) + assert_equal(import_info['hdkeypath'], priv_keypath) + + # Check legacy import does not import key origin info + self.log.info("Legacy imports don't have key origin info") + pub_addr = self.nodes[1].getnewaddress() + info = self.nodes[1].getaddressinfo(pub_addr) + pub = info['pubkey'] + result = self.nodes[0].importmulti( + [{ + 'scriptPubKey': {'address': pub_addr}, + 'pubkeys': [pub], + "timestamp": "now", + }] + ) + assert result[0]['success'] + pub_import_info = self.nodes[0].getaddressinfo(pub_addr) + assert_equal(pub_import_info['pubkey'], pub) + assert 'hdmasterfingerprint' not in pub_import_info + assert 'hdkeypath' not in pub_import_info + + # Import some public keys to the keypool of a no privkey wallet + self.log.info("Adding pubkey to keypool of disableprivkey wallet") + self.nodes[1].createwallet(wallet_name="noprivkeys", disable_private_keys=True) + wrpc = self.nodes[1].get_wallet_rpc("noprivkeys") + + addr1 = self.nodes[0].getnewaddress() + addr2 = self.nodes[0].getnewaddress() + pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] + pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] + result = wrpc.importmulti( + [{ + 'desc': 'wpkh(' + pub1 + ')', + 'keypool': True, + "timestamp": "now", + }, + { + 'desc': 'wpkh(' + pub2 + ')', + 'keypool': True, + "timestamp": "now", + }] + ) + assert result[0]['success'] + assert result[1]['success'] + assert_equal(wrpc.getwalletinfo()["keypoolsize"], 2) + newaddr1 = wrpc.getnewaddress() + assert_equal(addr1, newaddr1) + newaddr2 = wrpc.getnewaddress() + assert_equal(addr2, newaddr2) + + # Import some public keys to the internal keypool of a no privkey wallet + self.log.info("Adding pubkey to internal keypool of disableprivkey wallet") + addr1 = self.nodes[0].getnewaddress() + addr2 = self.nodes[0].getnewaddress() + pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] + pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] + result = wrpc.importmulti( + [{ + 'desc': 'wpkh(' + pub1 + ')', + 'keypool': True, + 'internal': True, + "timestamp": "now", + }, + { + 'desc': 'wpkh(' + pub2 + ')', + 'keypool': True, + 'internal': True, + "timestamp": "now", + }] + ) + assert result[0]['success'] + assert result[1]['success'] + assert_equal(wrpc.getwalletinfo()["keypoolsize_hd_internal"], 2) + newaddr1 = wrpc.getrawchangeaddress() + assert_equal(addr1, newaddr1) + newaddr2 = wrpc.getrawchangeaddress() + assert_equal(addr2, newaddr2) + + # Import a multisig and make sure the keys don't go into the keypool + self.log.info('Imported scripts with pubkeys shoud not have their pubkeys go into the keypool') + addr1 = self.nodes[0].getnewaddress() + addr2 = self.nodes[0].getnewaddress() + pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] + pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] + result = wrpc.importmulti( + [{ + 'desc': 'wsh(multi(2,' + pub1 + ',' + pub2 + '))', + 'keypool': True, + "timestamp": "now", + }] + ) + assert result[0]['success'] + assert_equal(wrpc.getwalletinfo()["keypoolsize"], 0) + + # Cannot import those pubkeys to keypool of wallet with privkeys + self.log.info("Pubkeys cannot be added to the keypool of a wallet with private keys") + wrpc = self.nodes[1].get_wallet_rpc("") + assert wrpc.getwalletinfo()['private_keys_enabled'] + result = wrpc.importmulti( + [{ + 'desc': 'wpkh(' + pub1 + ')', + 'keypool': True, + "timestamp": "now", + }] + ) + assert_equal(result[0]['error']['code'], -8) + assert_equal(result[0]['error']['message'], "Keys can only be imported to the keypool when private keys are disabled") + + # Make sure ranged imports import keys in order + self.log.info('Key ranges should be imported in order') + wrpc = self.nodes[1].get_wallet_rpc("noprivkeys") + assert_equal(wrpc.getwalletinfo()["keypoolsize"], 0) + assert_equal(wrpc.getwalletinfo()["private_keys_enabled"], False) + xpub = "tpubDAXcJ7s7ZwicqjprRaEWdPoHKrCS215qxGYxpusRLLmJuT69ZSicuGdSfyvyKpvUNYBW1s2U3NSrT6vrCYB9e6nZUEvrqnwXPF8ArTCRXMY" + addresses = [ + 'bcrt1qtmp74ayg7p24uslctssvjm06q5phz4yrxucgnv', # m/0'/0'/0 + 'bcrt1q8vprchan07gzagd5e6v9wd7azyucksq2xc76k8', # m/0'/0'/1 + 'bcrt1qtuqdtha7zmqgcrr26n2rqxztv5y8rafjp9lulu', # m/0'/0'/2 + 'bcrt1qau64272ymawq26t90md6an0ps99qkrse58m640', # m/0'/0'/3 + 'bcrt1qsg97266hrh6cpmutqen8s4s962aryy77jp0fg0', # m/0'/0'/4 + ] + result = wrpc.importmulti( + [{ + 'desc': 'wpkh([80002067/0h/0h]' + xpub + '/*)', + 'keypool': True, + 'timestamp': 'now', + 'range' : {'start': 0, 'end': 4} + }] + ) + for i in range(0, 5): + addr = wrpc.getnewaddress('', 'bech32') + assert_equal(addr, addresses[i]) if __name__ == '__main__': ImportMultiTest().main() diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index ceb9709712..1116196268 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -21,7 +21,6 @@ class KeyPoolTest(BitcoinTestFramework): addr_before_encrypting = nodes[0].getnewaddress() addr_before_encrypting_data = nodes[0].getaddressinfo(addr_before_encrypting) wallet_info_old = nodes[0].getwalletinfo() - assert_equal(wallet_info_old['hdseedid'], wallet_info_old['hdmasterkeyid']) assert(addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid']) # Encrypt wallet and wait to terminate @@ -30,7 +29,6 @@ class KeyPoolTest(BitcoinTestFramework): addr = nodes[0].getnewaddress() addr_data = nodes[0].getaddressinfo(addr) wallet_info = nodes[0].getwalletinfo() - assert_equal(wallet_info['hdseedid'], wallet_info['hdmasterkeyid']) assert(addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid']) assert(addr_data['hdseedid'] == wallet_info['hdseedid']) assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py new file mode 100755 index 0000000000..1869f71753 --- /dev/null +++ b/test/fuzz/test_runner.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Run fuzz test targets. +""" + +import argparse +import configparser +import os +import sys +import subprocess +import logging + + +def main(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + "-l", + "--loglevel", + dest="loglevel", + default="INFO", + help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.", + ) + parser.add_argument( + '--export_coverage', + action='store_true', + help='If true, export coverage information to files in the seed corpus', + ) + parser.add_argument( + 'seed_dir', + help='The seed corpus to run on (must contain subfolders for each fuzz target).', + ) + parser.add_argument( + 'target', + nargs='*', + help='The target(s) to run. Default is to run all targets.', + ) + + args = parser.parse_args() + + # Set up logging + logging.basicConfig( + format='%(message)s', + level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(), + ) + + # Read config generated by configure. + config = configparser.ConfigParser() + configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" + config.read_file(open(configfile, encoding="utf8")) + + if not config["components"].getboolean("ENABLE_FUZZ"): + logging.error("Must have fuzz targets built") + sys.exit(1) + + # Build list of tests + test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include')) + + if not test_list_all: + logging.error("No fuzz targets found") + sys.exit(1) + + logging.info("Fuzz targets found: {}".format(test_list_all)) + + args.target = args.target or test_list_all # By default run all + test_list_error = list(set(args.target).difference(set(test_list_all))) + if test_list_error: + logging.error("Unknown fuzz targets selected: {}".format(test_list_error)) + test_list_selection = list(set(test_list_all).intersection(set(args.target))) + if not test_list_selection: + logging.error("No fuzz targets selected") + logging.info("Fuzz targets selected: {}".format(test_list_selection)) + + try: + help_output = subprocess.run( + args=[ + os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]), + '-help=1', + ], + timeout=1, + check=True, + stderr=subprocess.PIPE, + universal_newlines=True, + ).stderr + if "libFuzzer" not in help_output: + logging.error("Must be built with libFuzzer") + sys.exit(1) + except subprocess.TimeoutExpired: + logging.error("subprocess timed out: Currently only libFuzzer is supported") + sys.exit(1) + + run_once( + corpus=args.seed_dir, + test_list=test_list_selection, + build_dir=config["environment"]["BUILDDIR"], + export_coverage=args.export_coverage, + ) + + +def run_once(*, corpus, test_list, build_dir, export_coverage): + for t in test_list: + args = [ + os.path.join(build_dir, 'src', 'test', 'fuzz', t), + '-runs=1', + os.path.join(corpus, t), + ] + logging.debug('Run {} with args {}'.format(t, args)) + output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr + logging.debug('Output: {}'.format(output)) + if not export_coverage: + continue + for l in output.splitlines(): + if 'INITED' in l: + with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file: + cov_file.write(l) + break + + +def parse_test_list(makefile): + with open(makefile, encoding='utf-8') as makefile_test: + test_list_all = [] + read_targets = False + for line in makefile_test.readlines(): + line = line.strip().replace('test/fuzz/', '').replace(' \\', '') + if read_targets: + if not line: + break + test_list_all.append(line) + continue + + if line == 'FUZZ_TARGETS =': + read_targets = True + return test_list_all + + +if __name__ == '__main__': + main() |