diff options
285 files changed, 7808 insertions, 4518 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index eb2d32e1bc..ac39fe235b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,18 +7,19 @@ clone_depth: 5 environment: APPVEYOR_SAVE_CACHE_ON_ERROR: true CLCACHE_SERVER: 1 - PACKAGES: boost-filesystem boost-signals2 boost-test libevent openssl zeromq berkeleydb + PACKAGES: berkeleydb boost-filesystem boost-signals2 boost-test libevent openssl rapidcheck zeromq PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' PYTHONUTF8: 1 cache: -- C:\tools\vcpkg\installed -- C:\Users\appveyor\clcache +- C:\tools\vcpkg\installed -> .appveyor.yml +- C:\Users\appveyor\clcache -> .appveyor.yml, build_msvc\**, **\Makefile.am, **\*.vcxproj.in install: - cmd: pip install --quiet git+https://github.com/frerich/clcache.git@v4.2.0 # Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. # - cmd: pip install zmq +- cmd: echo set(VCPKG_BUILD_TYPE release) >> C:\tools\vcpkg\triplets\%PLATFORM%-windows-static.cmake +- cmd: vcpkg remove --outdated --recurse - cmd: vcpkg install --triplet %PLATFORM%-windows-static %PACKAGES% > NUL -- cmd: del /s /q C:\Tools\vcpkg\installed\%PLATFORM%-windows-static\debug # Remove unused debug library before_build: - ps: clcache -M 536870912 - cmd: python build_msvc\msvc-autogen.py @@ -27,7 +28,6 @@ before_build: ${content} = (Get-Content ${files}[${i}]); ${content} = ${content}.Replace("</RuntimeLibrary>", "</RuntimeLibrary><DebugInformationFormat>None</DebugInformationFormat>"); ${content} = ${content}.Replace("<WholeProgramOptimization>true", "<WholeProgramOptimization>false"); - ${content} = ${content}.Replace("NDEBUG;", ""); Set-Content ${files}[${i}] ${content}; } - ps: Start-Process clcache-server @@ -54,5 +54,5 @@ test_script: - cmd: src\bench_bitcoin.exe -evals=1 -scaling=0 > NUL - ps: python test\util\bitcoin-util-test.py - cmd: python test\util\rpcauth-test.py -- cmd: python test\functional\test_runner.py --ci --force --quiet --combinedlogslen=4000 --failfast +- cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast deploy: off diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000000..9104a0a3d1 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,26 @@ +task: + name: "FreeBsd 12.0 amd64 [GOAL: install] [no depends, only system libs]" + freebsd_instance: + image: freebsd-12-0-release-amd64 + ccache_cache: + folder: "/tmp/ccache_dir" + env: + MAKEJOBS: "-j3" + CONFIGURE_OPTS: "--disable-dependency-tracking" + GOAL: "install" + CCACHE_SIZE: "200M" + CCACHE_COMPRESS: 1 + CCACHE_DIR: "/tmp/ccache_dir" + install_script: + - pkg install -y autoconf automake boost-libs git gmake libevent libtool openssl pkgconf python3 ccache + - ./contrib/install_db4.sh $(pwd) + - ccache --max-size=${CCACHE_SIZE} + configure_script: + - ./autogen.sh + - ./configure ${CONFIGURE_OPTS} BDB_LIBS="-L$(pwd)/db4/lib -ldb_cxx-4.8" BDB_CFLAGS="-I$(pwd)/db4/include" || ( cat config.log && false) + make_script: + - gmake ${MAKEJOBS} ${GOAL} || ( echo "Build failure. Verbose build follows." && gmake ${GOAL} V=1 ; false ) + check_script: + - gmake check ${MAKEJOBS} VERBOSE=1 + functional_test_script: + - ./test/functional/test_runner.py --ci --combinedlogslen=1000 --quiet --failfast diff --git a/.python-version b/.python-version index 7bcbb3808b..c49282585a 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.4.9 +3.5.6 diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000000..fe6227baf6 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,261 @@ +[style] +# Align closing bracket with visual indentation. +align_closing_bracket_with_visual_indent=True + +# Allow dictionary keys to exist on multiple lines. For example: +# +# x = { +# ('this is the first element of a tuple', +# 'this is the second element of a tuple'): +# value, +# } +allow_multiline_dictionary_keys=False + +# Allow lambdas to be formatted on more than one line. +allow_multiline_lambdas=False + +# Allow splits before the dictionary value. +allow_split_before_dict_value=True + +# Number of blank lines surrounding top-level function and class +# definitions. +blank_lines_around_top_level_definition=2 + +# Insert a blank line before a class-level docstring. +blank_line_before_class_docstring=False + +# Insert a blank line before a module docstring. +blank_line_before_module_docstring=False + +# Insert a blank line before a 'def' or 'class' immediately nested +# within another 'def' or 'class'. For example: +# +# class Foo: +# # <------ this blank line +# def method(): +# ... +blank_line_before_nested_class_or_def=False + +# Do not split consecutive brackets. Only relevant when +# dedent_closing_brackets is set. For example: +# +# call_func_that_takes_a_dict( +# { +# 'key1': 'value1', +# 'key2': 'value2', +# } +# ) +# +# would reformat to: +# +# call_func_that_takes_a_dict({ +# 'key1': 'value1', +# 'key2': 'value2', +# }) +coalesce_brackets=False + +# The column limit. +column_limit=79 + +# The style for continuation alignment. Possible values are: +# +# - SPACE: Use spaces for continuation alignment. This is default behavior. +# - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns +# (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs) for continuation +# alignment. +# - LESS: Slightly left if cannot vertically align continuation lines with +# indent characters. +# - VALIGN-RIGHT: Vertically align continuation lines with indent +# characters. Slightly right (one more indent character) if cannot +# vertically align continuation lines with indent characters. +# +# For options FIXED, and VALIGN-RIGHT are only available when USE_TABS is +# enabled. +continuation_align_style=SPACE + +# Indent width used for line continuations. +continuation_indent_width=4 + +# Put closing brackets on a separate line, dedented, if the bracketed +# expression can't fit in a single line. Applies to all kinds of brackets, +# including function definitions and calls. For example: +# +# config = { +# 'key1': 'value1', +# 'key2': 'value2', +# } # <--- this bracket is dedented and on a separate line +# +# time_series = self.remote_client.query_entity_counters( +# entity='dev3246.region1', +# key='dns.query_latency_tcp', +# transform=Transformation.AVERAGE(window=timedelta(seconds=60)), +# start_ts=now()-timedelta(days=3), +# end_ts=now(), +# ) # <--- this bracket is dedented and on a separate line +dedent_closing_brackets=False + +# Disable the heuristic which places each list element on a separate line +# if the list is comma-terminated. +disable_ending_comma_heuristic=False + +# Place each dictionary entry onto its own line. +each_dict_entry_on_separate_line=True + +# The regex for an i18n comment. The presence of this comment stops +# reformatting of that line, because the comments are required to be +# next to the string they translate. +i18n_comment= + +# The i18n function call names. The presence of this function stops +# reformattting on that line, because the string it has cannot be moved +# away from the i18n comment. +i18n_function_call= + +# Indent the dictionary value if it cannot fit on the same line as the +# dictionary key. For example: +# +# config = { +# 'key1': +# 'value1', +# 'key2': value1 + +# value2, +# } +indent_dictionary_value=False + +# The number of columns to use for indentation. +indent_width=4 + +# Join short lines into one line. E.g., single line 'if' statements. +join_multiple_lines=True + +# Do not include spaces around selected binary operators. For example: +# +# 1 + 2 * 3 - 4 / 5 +# +# will be formatted as follows when configured with "*,/": +# +# 1 + 2*3 - 4/5 +# +no_spaces_around_selected_binary_operators= + +# Use spaces around default or named assigns. +spaces_around_default_or_named_assign=False + +# Use spaces around the power operator. +spaces_around_power_operator=False + +# The number of spaces required before a trailing comment. +spaces_before_comment=2 + +# Insert a space between the ending comma and closing bracket of a list, +# etc. +space_between_ending_comma_and_closing_bracket=True + +# Split before arguments +split_all_comma_separated_values=False + +# Split before arguments if the argument list is terminated by a +# comma. +split_arguments_when_comma_terminated=False + +# Set to True to prefer splitting before '&', '|' or '^' rather than +# after. +split_before_bitwise_operator=True + +# Split before the closing bracket if a list or dict literal doesn't fit on +# a single line. +split_before_closing_bracket=True + +# Split before a dictionary or set generator (comp_for). For example, note +# the split before the 'for': +# +# foo = { +# variable: 'Hello world, have a nice day!' +# for variable in bar if variable != 42 +# } +split_before_dict_set_generator=True + +# Split before the '.' if we need to split a longer expression: +# +# foo = ('This is a really long string: {}, {}, {}, {}'.format(a, b, c, d)) +# +# would reformat to something like: +# +# foo = ('This is a really long string: {}, {}, {}, {}' +# .format(a, b, c, d)) +split_before_dot=False + +# Split after the opening paren which surrounds an expression if it doesn't +# fit on a single line. +split_before_expression_after_opening_paren=False + +# If an argument / parameter list is going to be split, then split before +# the first argument. +split_before_first_argument=False + +# Set to True to prefer splitting before 'and' or 'or' rather than +# after. +split_before_logical_operator=True + +# Split named assignments onto individual lines. +split_before_named_assigns=True + +# Set to True to split list comprehensions and generators that have +# non-trivial expressions and multiple clauses before each of these +# clauses. For example: +# +# result = [ +# a_long_var + 100 for a_long_var in xrange(1000) +# if a_long_var % 10] +# +# would reformat to something like: +# +# result = [ +# a_long_var + 100 +# for a_long_var in xrange(1000) +# if a_long_var % 10] +split_complex_comprehension=False + +# The penalty for splitting right after the opening bracket. +split_penalty_after_opening_bracket=30 + +# The penalty for splitting the line after a unary operator. +split_penalty_after_unary_operator=10000 + +# The penalty for splitting right before an if expression. +split_penalty_before_if_expr=0 + +# The penalty of splitting the line around the '&', '|', and '^' +# operators. +split_penalty_bitwise_operator=300 + +# The penalty for splitting a list comprehension or generator +# expression. +split_penalty_comprehension=80 + +# The penalty for characters over the column limit. +split_penalty_excess_character=7000 + +# The penalty incurred by adding a line split to the unwrapped line. The +# more line splits added the higher the penalty. +split_penalty_for_added_line_split=30 + +# The penalty of splitting a list of "import as" names. For example: +# +# from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, +# long_argument_2, +# long_argument_3) +# +# would reformat to something like: +# +# from a_very_long_or_indented_module_name_yada_yad import ( +# long_argument_1, long_argument_2, long_argument_3) +split_penalty_import_names=0 + +# The penalty of splitting the line around the 'and' and 'or' +# operators. +split_penalty_logical_operator=300 + +# Use the Tab character for indentation. +use_tabs=False + diff --git a/.travis.yml b/.travis.yml index 873b33b05b..fea6b6d7bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ env: - MAKEJOBS=-j3 - RUN_UNIT_TESTS=true - RUN_FUNCTIONAL_TESTS=true + - RUN_FUZZ_TESTS=false - DOCKER_NAME_TAG=ubuntu:18.04 - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID - CCACHE_SIZE=100M @@ -45,7 +46,7 @@ jobs: env: cache: false language: python - python: '3.4' # Oldest supported version according to doc/dependencies.md + python: '3.5' # Oldest supported version according to doc/dependencies.md install: - set -o errexit; source .travis/lint_04_install.sh before_script: @@ -66,29 +67,31 @@ jobs: BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi" - stage: test - name: 'Win32 [GOAL: deploy] [no gui tests]' + name: 'Win32 [GOAL: deploy] [no gui or functional tests]' env: >- HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" PACKAGES="python3 nsis g++-mingw-w64-i686 wine-binfmt wine32" + RUN_FUNCTIONAL_TESTS=false GOAL="deploy" BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" - stage: test - name: 'Win64 [GOAL: deploy] [no gui tests]' + name: 'Win64 [GOAL: deploy] [no gui or functional tests]' env: >- HOST=x86_64-w64-mingw32 PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" + RUN_FUNCTIONAL_TESTS=false GOAL="deploy" BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests" - stage: test - name: '32-bit + dash [GOAL: install]' + name: '32-bit + dash [GOAL: install] [GUI: no BIP70]' env: >- HOST=i686-pc-linux-gnu PACKAGES="g++-multilib python3-zmq" GOAL="install" - BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" + BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --disable-bip70 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" CONFIG_SHELL="/bin/dash" - stage: test @@ -98,7 +101,7 @@ jobs: PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" GOAL="install" - BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-fuzz --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\"" + BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\"" - stage: test name: 'x86_64 Linux [GOAL: install] [trusty] [no functional tests, no depends, only system libs]' @@ -131,6 +134,18 @@ jobs: BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" - stage: test + name: 'x86_64 Linux [GOAL: install] [bionic] [no depends, only system libs, sanitizers: fuzzer,address]' + env: >- + HOST=x86_64-unknown-linux-gnu + PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev" + NO_DEPENDS=1 + RUN_UNIT_TESTS=false + RUN_FUNCTIONAL_TESTS=false + RUN_FUZZ_TESTS=true + GOAL="install" + BITCOIN_CONFIG="--disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++" + + - stage: test name: 'x86_64 Linux [GOAL: install] [bionic] [no wallet]' env: >- HOST=x86_64-unknown-linux-gnu @@ -143,7 +158,7 @@ jobs: name: 'macOS 10.10 [GOAL: deploy] [no functional tests]' env: >- HOST=x86_64-apple-darwin14 - PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" + PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools-git" OSX_SDK=10.11 RUN_UNIT_TESTS=false RUN_FUNCTIONAL_TESTS=false diff --git a/.travis/test_04_install.sh b/.travis/test_04_install.sh index a111387f10..8055bbdd19 100755 --- a/.travis/test_04_install.sh +++ b/.travis/test_04_install.sh @@ -7,6 +7,11 @@ export LC_ALL=C.UTF-8 travis_retry docker pull "$DOCKER_NAME_TAG" + +export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets +git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN} +export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/ + mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/" export ASAN_OPTIONS="" export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan" diff --git a/.travis/test_06_script_b.sh b/.travis/test_06_script_b.sh index b6b5925be8..e13abfd52f 100755 --- a/.travis/test_06_script_b.sh +++ b/.travis/test_06_script_b.sh @@ -19,3 +19,9 @@ if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then DOCKER_EXEC test/functional/test_runner.py --ci --combinedlogslen=4000 --coverage --quiet --failfast END_FOLD fi + +if [ "$RUN_FUZZ_TESTS" = "true" ]; then + BEGIN_FOLD fuzz-tests + DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN} + END_FOLD +fi diff --git a/Makefile.am b/Makefile.am index 9a6e15f0dd..85674f819a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -20,6 +20,7 @@ BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT) BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT) BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT) BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT) +BITCOIN_WALLET_BIN=$(top_builddir)/src/$(BITCOIN_WALLET_TOOL_NAME)$(EXEEXT) BITCOIN_WIN_INSTALLER=$(PACKAGE)-$(PACKAGE_VERSION)-win$(WINDOWS_BITS)-setup$(EXEEXT) empty := @@ -76,6 +77,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release + STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release @test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \ echo error: could not build $@ @echo built $@ @@ -172,6 +174,9 @@ $(BITCOIN_CLI_BIN): FORCE $(BITCOIN_TX_BIN): FORCE $(MAKE) -C src $(@F) +$(BITCOIN_WALLET_BIN): FORCE + $(MAKE) -C src $(@F) + if USE_LCOV LCOV_FILTER_PATTERN=-p "/usr/include/" -p "/usr/lib/" -p "src/leveldb/" -p "src/bench/" -p "src/univalue" -p "src/crypto/ctaes" -p "src/secp256k1" @@ -220,7 +225,11 @@ endif dist_noinst_SCRIPTS = autogen.sh -EXTRA_DIST = $(DIST_SHARE) test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) +EXTRA_DIST = $(DIST_SHARE) $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS) + +EXTRA_DIST += \ + test/functional \ + test/fuzz EXTRA_DIST += \ test/util/bitcoin-util-test.py \ @@ -66,7 +66,7 @@ Translations ------------ Changes to translations as well as new translations can be submitted to -[Bitcoin Core's Transifex page](https://www.transifex.com/projects/p/bitcoin/). +[Bitcoin Core's Transifex page](https://www.transifex.com/bitcoin/bitcoin/). Translations are periodically pulled from Transifex and merged into the git repository. See the [translation process](doc/translation_process.md) for details on how this works. diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 90a2cd9875..1a7c5d5f7d 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -116,24 +116,6 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ if test "x$bitcoin_cv_static_qt" = xyes; then _BITCOIN_QT_FIND_STATIC_PLUGINS AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static]) - AC_CACHE_CHECK(for Qt < 5.4, bitcoin_cv_need_acc_widget,[ - AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include <QtCore/qconfig.h> - #ifndef QT_VERSION - # include <QtCore/qglobal.h> - #endif - ]], - [[ - #if QT_VERSION >= 0x050400 - choke - #endif - ]])], - [bitcoin_cv_need_acc_widget=yes], - [bitcoin_cv_need_acc_widget=no]) - ]) - if test "x$bitcoin_cv_need_acc_widget" = xyes; then - _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(AccessibleFactory)], [-lqtaccessiblewidgets]) - fi _BITCOIN_QT_CHECK_STATIC_PLUGINS([Q_IMPORT_PLUGIN(QMinimalIntegrationPlugin)],[-lqminimal]) AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) if test "x$TARGET_OS" = xwindows; then @@ -264,7 +246,7 @@ dnl All macros below are internal and should _not_ be used from the main dnl configure.ac. dnl ---- -dnl Internal. Check if the included version of Qt is Qt5. +dnl Internal. Check included version of Qt against minimum specified in doc/dependencies.md dnl Requires: INCLUDES must be populated as necessary. dnl Output: bitcoin_cv_qt5=yes|no AC_DEFUN([_BITCOIN_QT_CHECK_QT5],[ @@ -276,7 +258,7 @@ AC_DEFUN([_BITCOIN_QT_CHECK_QT5],[ #endif ]], [[ - #if QT_VERSION < 0x050200 || QT_VERSION_MAJOR < 5 + #if QT_VERSION < 0x050501 choke #endif ]])], @@ -374,9 +356,7 @@ AC_DEFUN([_BITCOIN_QT_FIND_STATIC_PLUGINS],[ fi if test "x$TARGET_OS" = xlinux; then PKG_CHECK_MODULES([X11XCB], [x11-xcb], [QT_LIBS="$X11XCB_LIBS $QT_LIBS"]) - if ${PKG_CONFIG} --exists "Qt5Core >= 5.5" 2>/dev/null; then - PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) - fi + PKG_CHECK_MODULES([QTXCBQPA], [Qt5XcbQpa], [QT_LIBS="$QTXCBQPA_LIBS $QT_LIBS"]) elif test "x$TARGET_OS" = xdarwin; then PKG_CHECK_MODULES([QTCLIPBOARD], [Qt5ClipboardSupport], [QT_LIBS="-lQt5ClipboardSupport $QT_LIBS"]) PKG_CHECK_MODULES([QTGRAPHICS], [Qt5GraphicsSupport], [QT_LIBS="-lQt5GraphicsSupport $QT_LIBS"]) @@ -527,4 +507,3 @@ AC_DEFUN([_BITCOIN_QT_FIND_LIBS_WITHOUT_PKGCONFIG],[ CXXFLAGS="$TEMP_CXXFLAGS" LIBS="$TEMP_LIBS" ]) - diff --git a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj index 771e8a56f4..0723243cd7 100644 --- a/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj +++ b/build_msvc/bench_bitcoin/bench_bitcoin.vcxproj @@ -32,6 +32,7 @@ <ClCompile Include="..\..\src\bench\examples.cpp" /> <ClCompile Include="..\..\src\bench\lockedpool.cpp" /> <ClCompile Include="..\..\src\bench\mempool_eviction.cpp" /> + <ClCompile Include="..\..\src\bench\rpc_mempool.cpp" /> <ClCompile Include="..\..\src\bench\merkle_root.cpp" /> <ClCompile Include="..\..\src\bench\rollingbloom.cpp" /> <ClCompile Include="..\..\src\bench\verify_script.cpp" /> @@ -146,7 +147,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>NOMINMAX;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -164,7 +164,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -180,7 +179,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -198,7 +196,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>NOMINMAX;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj index 32f0354fad..6b668054a3 100644 --- a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj +++ b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj @@ -111,9 +111,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -131,7 +130,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -147,7 +145,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -163,9 +160,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj b/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj index 86fd614990..1d0174e319 100644 --- a/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj +++ b/build_msvc/bitcoin-tx/bitcoin-tx.vcxproj @@ -114,9 +114,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -134,7 +133,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -150,7 +148,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -166,9 +163,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj index 367e68ee51..fa86da7bae 100644 --- a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj +++ b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj @@ -56,6 +56,9 @@ <ProjectReference Include="..\libsecp256k1\libsecp256k1.vcxproj"> <Project>{bb493552-3b8c-4a8c-bf69-a6e7a51d2ea6}</Project> </ProjectReference> + <ProjectReference Include="..\libleveldb\libleveldb.vcxproj"> + <Project>{18430fef-6b61-4c53-b396-718e02850f1b}</Project> + </ProjectReference> </ItemGroup> <PropertyGroup Label="Globals"> <VCProjectVersion>15.0</VCProjectVersion> @@ -129,9 +132,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -149,7 +151,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -165,7 +166,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -181,9 +181,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index 9a071f3bdd..bb212af52e 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -79,7 +79,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -96,7 +95,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -113,9 +111,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -136,9 +133,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/common.vcxproj b/build_msvc/common.vcxproj index 93004dda7c..889dc6c2ad 100644 --- a/build_msvc/common.vcxproj +++ b/build_msvc/common.vcxproj @@ -24,6 +24,7 @@ <DisableSpecificWarnings>4018;4244;4267;4715;4805;</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> <PreprocessorDefinitions>_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> </ItemDefinitionGroup> </Project> diff --git a/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in b/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in index b7265054fb..5849e463a6 100644 --- a/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in +++ b/build_msvc/libbitcoin_cli/libbitcoin_cli.vcxproj.in @@ -97,9 +97,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -116,7 +115,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -131,7 +129,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -146,9 +143,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in index b51cdb2991..292e193f5d 100644 --- a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in +++ b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in @@ -91,7 +91,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> @@ -109,7 +108,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> @@ -127,9 +125,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> @@ -149,9 +146,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> diff --git a/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in b/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in index a05125723a..e7002036ad 100644 --- a/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in +++ b/build_msvc/libbitcoin_crypto/libbitcoin_crypto.vcxproj.in @@ -89,7 +89,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -104,7 +103,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -119,9 +117,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -138,9 +135,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 43bfb271fb..73ba90aa88 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -873,7 +873,7 @@ <Optimization>Disabled</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>false</IntrinsicFunctions> - <PreprocessorDefinitions>_X86_;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_X86_;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>.\GeneratedFiles;..\..\src;..\..\src\univalue\include;.\QtGenerated\mocheaders</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -922,7 +922,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>.\GeneratedFiles;..\..\src;..\..\src\univalue\include;.\QtGenerated\mocheaders</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in b/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in index acb827bd95..48a117bcfe 100644 --- a/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in +++ b/build_msvc/libbitcoin_server/libbitcoin_server.vcxproj.in @@ -89,9 +89,8 @@ <PrecompiledHeader>NotUsing</PrecompiledHeader> <WarningLevel>Level3</WarningLevel> <Optimization>Disabled</Optimization> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -104,9 +103,8 @@ <PrecompiledHeader>NotUsing</PrecompiledHeader> <WarningLevel>Level3</WarningLevel> <Optimization>Disabled</Optimization> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -122,9 +120,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -141,9 +138,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in index dc17c98e98..dbd91cf4db 100644 --- a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in +++ b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in @@ -92,7 +92,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -112,7 +111,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -132,9 +130,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -156,9 +153,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in index 1bb7be6f7f..33f4054546 100644 --- a/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in +++ b/build_msvc/libbitcoin_wallet/libbitcoin_wallet.vcxproj.in @@ -97,7 +97,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\wallet;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions> </ClCompile> @@ -113,7 +112,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\wallet;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions> </ClCompile> @@ -129,9 +127,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\wallet;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions> </ClCompile> @@ -149,9 +146,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\wallet;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <AdditionalOptions>/bigobj %(AdditionalOptions)</AdditionalOptions> </ClCompile> diff --git a/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in b/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in index b9cf68aee0..187d955687 100644 --- a/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in +++ b/build_msvc/libbitcoin_wallet_tool/libbitcoin_wallet_tool.vcxproj.in @@ -97,7 +97,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -113,7 +112,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -129,9 +127,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -149,9 +146,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> diff --git a/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in b/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in index e396c1ad0c..c877a280c0 100644 --- a/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in +++ b/build_msvc/libbitcoin_zmq/libbitcoin_zmq.vcxproj.in @@ -97,7 +97,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -113,7 +112,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -129,9 +127,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> @@ -149,9 +146,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>NOMINMAX;ZMQ_STATIC;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> <SuppressStartupBanner>false</SuppressStartupBanner> </ClCompile> diff --git a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj b/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj index 961225f9ba..a32aafbd74 100644 --- a/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj +++ b/build_msvc/libbitcoinconsensus/libbitcoinconsensus.vcxproj @@ -126,7 +126,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>false</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <ExceptionHandling>Sync</ExceptionHandling> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> @@ -143,7 +142,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>false</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <ExceptionHandling>Sync</ExceptionHandling> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> @@ -160,9 +158,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>false</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <ExceptionHandling>Sync</ExceptionHandling> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -181,9 +178,8 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>false</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\secp256k1\include;</AdditionalIncludeDirectories> <ExceptionHandling>Sync</ExceptionHandling> <SuppressStartupBanner>false</SuppressStartupBanner> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libleveldb/libleveldb.vcxproj b/build_msvc/libleveldb/libleveldb.vcxproj index 6de3a2b282..545508001e 100644 --- a/build_msvc/libleveldb/libleveldb.vcxproj +++ b/build_msvc/libleveldb/libleveldb.vcxproj @@ -162,7 +162,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NOMINMAX;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NOMINMAX;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\leveldb;..\..\src\leveldb\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -181,7 +181,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NOMINMAX;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CRT_NONSTDC_NO_DEPRECATE;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;NOMINMAX;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\leveldb;..\..\src\leveldb\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libsecp256k1/libsecp256k1.vcxproj b/build_msvc/libsecp256k1/libsecp256k1.vcxproj index 699c7d6871..b4c9ec28ee 100644 --- a/build_msvc/libsecp256k1/libsecp256k1.vcxproj +++ b/build_msvc/libsecp256k1/libsecp256k1.vcxproj @@ -122,7 +122,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;HAVE_CONFIG_H;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;HAVE_CONFIG_H;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\secp256k1;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -141,7 +141,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;HAVE_CONFIG_H;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>ENABLE_MODULE_ECDH;ENABLE_MODULE_RECOVERY;HAVE_CONFIG_H;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\secp256k1;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/libunivalue/libunivalue.vcxproj b/build_msvc/libunivalue/libunivalue.vcxproj index 57d469debf..c3799b6408 100644 --- a/build_msvc/libunivalue/libunivalue.vcxproj +++ b/build_msvc/libunivalue/libunivalue.vcxproj @@ -125,7 +125,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> @@ -144,7 +144,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> <AdditionalIncludeDirectories>..\..\src\univalue\include;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index b2d4c118f3..a5d666c114 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -89,7 +89,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> </ClCompile> <Link> @@ -132,7 +132,7 @@ <Optimization>MaxSpeed</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> </ClCompile> <Link> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 8f39132a22..a08e708f81 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -21,9 +21,11 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\..\src\test\*_tests.cpp" /> + <ClCompile Include="..\..\src\test\*_properties.cpp" /> + <ClCompile Include="..\..\src\test\gen\*_gen.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> <ClCompile Include="..\..\src\test\test_bitcoin.cpp" /> - <ClCompile Include="..\..\src\test\test_bitcoin_main.cpp" /> + <ClCompile Include="..\..\src\test\main.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_fixture.cpp" /> </ItemGroup> <ItemGroup> @@ -133,7 +135,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\test;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -150,7 +151,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\test;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -168,7 +168,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\test;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -188,7 +187,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NOMINMAX;WIN32;HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\leveldb\include;..\..\src\test;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/build_msvc/testconsensus/testconsensus.vcxproj b/build_msvc/testconsensus/testconsensus.vcxproj index db2f8a6216..a2f5659049 100644 --- a/build_msvc/testconsensus/testconsensus.vcxproj +++ b/build_msvc/testconsensus/testconsensus.vcxproj @@ -95,7 +95,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_SCL_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src\;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -110,7 +109,6 @@ <Optimization>Disabled</Optimization> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src\;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> </ClCompile> <Link> @@ -129,7 +127,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;_SCL_SECURE_NO_WARNINGS;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src\;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> @@ -148,7 +145,6 @@ <IntrinsicFunctions>true</IntrinsicFunctions> <PreprocessorDefinitions>HAVE_CONFIG_H;WIN32;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <SDLCheck>true</SDLCheck> - <AdditionalIncludeDirectories>..\..\src\;</AdditionalIncludeDirectories> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <Link> diff --git a/configure.ac b/configure.ac index ddd6c7f8eb..a3ba8ce808 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ dnl require autoconf 2.60 (AS_ECHO/AS_ECHO_N) AC_PREREQ([2.60]) define(_CLIENT_VERSION_MAJOR, 0) -define(_CLIENT_VERSION_MINOR, 17) +define(_CLIENT_VERSION_MINOR, 18) define(_CLIENT_VERSION_REVISION, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) @@ -85,8 +85,8 @@ AC_PATH_TOOL(RANLIB, ranlib) AC_PATH_TOOL(STRIP, strip) AC_PATH_TOOL(GCOV, gcov) AC_PATH_PROG(LCOV, lcov) -dnl Python 3.x is supported from 3.4 on (see https://github.com/bitcoin/bitcoin/issues/7893) -AC_PATH_PROGS([PYTHON], [python3.7 python3.6 python3.5 python3.4 python3 python]) +dnl Python 3.5 is specified in .python-version and should be used if available, see doc/dependencies.md +AC_PATH_PROGS([PYTHON], [python3.5 python3.6 python3.7 python3.8 python3 python]) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 6ee65f40be..d66eff66be 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -167,7 +167,7 @@ still compatible with the minimum supported Linux distribution versions. Example usage after a gitian build: - find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py + find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py If only supported symbols are used the return value will be 0 and the output will be empty. diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index abfa5ed5ae..2e4657f1dd 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -8,9 +8,18 @@ MAPPING = { 'core_write.cpp': 'core_io.cpp', } +# Directories with header-based modules, where the assumption that .cpp files +# define functions and variables declared in corresponding .h files is +# incorrect. +HEADER_MODULE_PATHS = [ + 'interfaces/' +] + def module_name(path): if path in MAPPING: path = MAPPING[path] + if any(path.startswith(dirpath) for dirpath in HEADER_MODULE_PATHS): + return path if path.endswith(".h"): return path[:-2] if path.endswith(".c"): diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index b5de5a395f..63b9847100 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -10,6 +10,7 @@ MANDIR=${MANDIR:-$TOPDIR/doc/man} BITCOIND=${BITCOIND:-$BINDIR/bitcoind} BITCOINCLI=${BITCOINCLI:-$BINDIR/bitcoin-cli} BITCOINTX=${BITCOINTX:-$BINDIR/bitcoin-tx} +WALLET_TOOL=${WALLET_TOOL:-$BINDIR/bitcoin-wallet} BITCOINQT=${BITCOINQT:-$BINDIR/qt/bitcoin-qt} [ ! -x $BITCOIND ] && echo "$BITCOIND not found or not executable." && exit 1 @@ -23,7 +24,7 @@ BTCVER=($($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ print $6, $7 }')) echo "[COPYRIGHT]" > footer.h2m $BITCOIND --version | sed -n '1!p' >> footer.h2m -for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $BITCOINQT; do +for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINQT; do cmdname="${cmd##*/}" help2man -N --version-string=${BTCVER[0]} --include=footer.h2m -o ${MANDIR}/${cmdname}.1 ${cmd} sed -i "s/\\\-${BTCVER[1]}//g" ${MANDIR}/${cmdname}.1 diff --git a/contrib/devtools/split-debug.sh.in b/contrib/devtools/split-debug.sh.in index deda49cc54..92b72b1446 100644 --- a/contrib/devtools/split-debug.sh.in +++ b/contrib/devtools/split-debug.sh.in @@ -1,5 +1,5 @@ #!/bin/sh - +set -e if [ $# -ne 3 ]; then echo "usage: $0 <input> <stripped-binary> <debug-binary>" fi diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index c6158c9422..7729dd7257 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -9,7 +9,7 @@ still compatible with the minimum supported Linux distribution versions. Example usage: - find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py + find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' import subprocess import re diff --git a/contrib/devtools/test_deterministic_coverage.sh b/contrib/devtools/test_deterministic_coverage.sh new file mode 100755 index 0000000000..16d03e1fff --- /dev/null +++ b/contrib/devtools/test_deterministic_coverage.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Test for deterministic coverage across unit test runs. + +export LC_ALL=C + +# Use GCOV_EXECUTABLE="gcov" if compiling with gcc. +# Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang. +GCOV_EXECUTABLE="gcov" + +# Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism. +NON_DETERMINISTIC_TESTS=( + "coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2)) + "denialofservice_tests/DoS_mapOrphans" # denialofservice_tests.cpp: it = mapOrphanTransactions.lower_bound(InsecureRand256()); + "fs_tests/fsbridge_fstream" # deterministic test failure? + "miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue() + "scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue() + "tx_validationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "tx_validationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txindex_tests/txindex_initial_sync" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/coin_mark_dirty_immature_credit" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/dummy_input_size_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/importmulti_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/importwallet_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/ListCoins" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/scan_for_wallet_transactions" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) + "wallet_tests/wallet_disableprivkeys" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10) +) + +TEST_BITCOIN_BINARY="src/test/test_bitcoin" + +print_usage() { + echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]" +} + +N_TEST_RUNS=2 +BOOST_TEST_RUN_FILTERS="" +if [[ $# != 0 ]]; then + if [[ $1 == "--help" ]]; then + print_usage + exit + fi + PARSED_ARGUMENTS=0 + if [[ $1 =~ [a-z] ]]; then + BOOST_TEST_RUN_FILTERS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ $1 =~ ^[0-9]+$ ]]; then + N_TEST_RUNS=$1 + PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1)) + shift + fi + if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then + print_usage + exit + fi +fi +if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then + BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')" +else + echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}" + echo +fi + +if ! command -v gcov > /dev/null; then + echo "Error: gcov not installed. Exiting." + exit 1 +fi + +if ! command -v gcovr > /dev/null; then + echo "Error: gcovr not installed. Exiting." + exit 1 +fi + +if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then + echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"./configure --enable-lcov\" and compile." + exit 1 +fi + +get_file_suffix_count() { + find src/ -type f -name "*.$1" | wc -l +} + +if [[ $(get_file_suffix_count gcno) == 0 ]]; then + echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"./configure --enable-lcov\" and re-compile." + exit 1 +fi + +get_covr_filename() { + echo "gcovr.run-$1.txt" +} + +TEST_RUN_ID=0 +while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do + TEST_RUN_ID=$((TEST_RUN_ID + 1)) + echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}" + find src/ -type f -name "*.gcda" -exec rm {} \; + if [[ $(get_file_suffix_count gcda) != 0 ]]; then + echo "Error: Stale *.gcda files found. Exiting." + exit 1 + fi + TEST_OUTPUT_TEMPFILE=$(mktemp) + if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then + cat "${TEST_OUTPUT_TEMPFILE}" + rm "${TEST_OUTPUT_TEMPFILE}" + exit 1 + fi + rm "${TEST_OUTPUT_TEMPFILE}" + if [[ $(get_file_suffix_count gcda) == 0 ]]; then + echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"./configure --enable-lcov\" and re-compile." + exit 1 + fi + GCOVR_TEMPFILE=$(mktemp) + if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then + echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting." + exit 1 + fi + GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID}) + mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}" + if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then + echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)." + exit 1 + fi + if [[ ${TEST_RUN_ID} != 1 ]]; then + COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}") + if [[ ${COVERAGE_DIFF} != "" ]]; then + echo + echo "The line coverage is non-deterministic between runs. Exiting." + echo + echo "The test suite must be deterministic in the sense that the set of lines executed at least" + echo "once must be identical between runs. This is a necessary condition for meaningful" + echo "coverage measuring." + echo + echo "${COVERAGE_DIFF}" + exit 1 + fi + rm "${GCOVR_FILENAME}" + fi +done + +echo +echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs." +exit diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index e97072c80a..43612d4b36 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-linux-0.18" +name: "bitcoin-core-linux-0.19" enable_cache: true suites: - "bionic" @@ -30,12 +30,13 @@ packages: - "faketime" - "bsdmainutils" - "ca-certificates" -- "python" +- "python3" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: [] script: | + set -e -o pipefail WRAP_DIR=$HOME/wrapped HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu" @@ -179,8 +180,8 @@ script: | find . -name "lib*.la" -delete find . -name "lib*.a" -delete rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME}/bin -type f -executable -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; - find ${DISTNAME}/lib -type f -exec ../contrib/devtools/split-debug.sh {} {} {}.dbg \; + find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg + find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg cp ../doc/README.md ${DISTNAME}/ find ${DISTNAME} -not -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index 297a136fae..df6cddbf03 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -12,6 +12,8 @@ remotes: files: - "bitcoin-osx-unsigned.tar.gz" script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped mkdir -p ${WRAP_DIR} export PATH=`pwd`:$PATH diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 87d8007ccb..2d18d0a359 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-osx-0.18" +name: "bitcoin-core-osx-0.19" enable_cache: true suites: - "bionic" @@ -23,9 +23,9 @@ packages: - "libcap-dev" - "libz-dev" - "libbz2-dev" -- "python" -- "python-dev" -- "python-setuptools" +- "python3" +- "python3-dev" +- "python3-setuptools" - "fonts-tuffy" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" @@ -33,6 +33,8 @@ remotes: files: - "MacOSX10.11.sdk.tar.gz" script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin14" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage" diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml index 045be873e9..08d20177a6 100644 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ b/contrib/gitian-descriptors/gitian-win-signer.yml @@ -16,6 +16,8 @@ files: - "osslsigncode-Backports-to-1.7.1.patch" - "bitcoin-win-unsigned.tar.gz" script: | + set -e -o pipefail + BUILD_DIR=`pwd` SIGDIR=${BUILD_DIR}/signature/win UNSIGNED_DIR=${BUILD_DIR}/unsigned diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 31b9c309c7..b805929035 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-win-0.18" +name: "bitcoin-core-win-0.19" enable_cache: true suites: - "bionic" @@ -20,13 +20,15 @@ packages: - "nsis" - "zip" - "ca-certificates" -- "python" +- "python3" - "rename" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" files: [] script: | + set -e -o pipefail + WRAP_DIR=$HOME/wrapped HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests" diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index e90b8af7e1..088d1c9dce 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -6,7 +6,7 @@ export LC_ALL=C set -e if [ -z "${1}" ]; then - echo "Usage: ./install_db4.sh <base-dir> [<extra-bdb-configure-flag> ...]" + echo "Usage: $0 <base-dir> [<extra-bdb-configure-flag> ...]" echo echo "Must specify a single argument: the directory in which db4 will be built." echo "This is probably \`pwd\` if you're at the root of the bitcoin repository." diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 2985106982..25a1c7351a 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -1,6 +1,5 @@ # Linearize -Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts -run using Python 3 but are compatible with Python 2. +Construct a linear, no-fork, best version of the Bitcoin blockchain. ## Step 1: Download hash list diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 56c1fbfc92..468aec04b5 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -16,7 +16,7 @@ import hashlib import datetime import time from collections import namedtuple -from binascii import hexlify, unhexlify +from binascii import unhexlify settings = {} @@ -61,7 +61,7 @@ def calc_hash_str(blk_hdr): hash = calc_hdr_hash(blk_hdr) hash = bufreverse(hash) hash = wordreverse(hash) - hash_str = hexlify(hash).decode('utf-8') + hash_str = hash.hex() return hash_str def get_blk_dt(blk_hdr): @@ -213,7 +213,7 @@ class BlockDataCopier: inMagic = inhdr[:4] if (inMagic != self.settings['netmagic']): - print("Invalid magic: " + hexlify(inMagic).decode('utf-8')) + print("Invalid magic: " + inMagic.hex()) return inLenLE = inhdr[4:] su = struct.unpack("<I", inLenLE) diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index 27ca15acb4..1215962a16 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -37,7 +37,8 @@ Configuration files Import trusted keys ------------------- -In order to check the commit signatures you must add the trusted PGP keys to your machine. This can be done in Linux by running +In order to check the commit signatures, you must add the trusted PGP keys to your machine. [GnuPG](https://gnupg.org/) may be used to import the trusted keys by running the following command: + ```sh gpg --recv-keys $(<contrib/verify-commits/trusted-keys) ``` diff --git a/contrib/zmq/zmq_sub3.4.py b/contrib/zmq/zmq_sub3.4.py deleted file mode 100644 index 66fdf7887f..0000000000 --- a/contrib/zmq/zmq_sub3.4.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -""" - ZMQ example using python3's asyncio - - Bitcoin should be started with the command line arguments: - bitcoind -testnet -daemon \ - -zmqpubrawtx=tcp://127.0.0.1:28332 \ - -zmqpubrawblock=tcp://127.0.0.1:28332 \ - -zmqpubhashtx=tcp://127.0.0.1:28332 \ - -zmqpubhashblock=tcp://127.0.0.1:28332 - - We use the asyncio library here. `self.handle()` installs itself as a - future at the end of the function. Since it never returns with the event - loop having an empty stack of futures, this creates an infinite loop. An - alternative is to wrap the contents of `handle` inside `while True`. - - The `@asyncio.coroutine` decorator and the `yield from` syntax found here - was introduced in python 3.4 and has been deprecated in favor of the `async` - and `await` keywords respectively. - - A blocking example using python 2.7 can be obtained from the git history: - https://github.com/bitcoin/bitcoin/blob/37a7fe9e440b83e2364d5498931253937abe9294/contrib/zmq/zmq_sub.py -""" - -import binascii -import asyncio -import zmq -import zmq.asyncio -import signal -import struct -import sys - -if (sys.version_info.major, sys.version_info.minor) < (3, 4): - print("This example only works with Python 3.4 and greater") - sys.exit(1) - -port = 28332 - -class ZMQHandler(): - def __init__(self): - self.loop = asyncio.get_event_loop() - self.zmqContext = zmq.asyncio.Context() - - self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) - self.zmqSubSocket.setsockopt(zmq.RCVHWM, 0) - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashblock") - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashtx") - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "rawblock") - self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "rawtx") - self.zmqSubSocket.connect("tcp://127.0.0.1:%i" % port) - - @asyncio.coroutine - def handle(self) : - msg = yield from self.zmqSubSocket.recv_multipart() - topic = msg[0] - body = msg[1] - sequence = "Unknown" - if len(msg[-1]) == 4: - msgSequence = struct.unpack('<I', msg[-1])[-1] - sequence = str(msgSequence) - if topic == b"hashblock": - print('- HASH BLOCK ('+sequence+') -') - print(binascii.hexlify(body)) - elif topic == b"hashtx": - print('- HASH TX ('+sequence+') -') - print(binascii.hexlify(body)) - elif topic == b"rawblock": - print('- RAW BLOCK HEADER ('+sequence+') -') - print(binascii.hexlify(body[:80])) - elif topic == b"rawtx": - print('- RAW TX ('+sequence+') -') - print(binascii.hexlify(body)) - # schedule ourselves to receive the next message - asyncio.ensure_future(self.handle()) - - def start(self): - self.loop.add_signal_handler(signal.SIGINT, self.stop) - self.loop.create_task(self.handle()) - self.loop.run_forever() - - def stop(self): - self.loop.stop() - self.zmqContext.destroy() - -daemon = ZMQHandler() -daemon.start() diff --git a/depends/README.md b/depends/README.md index 693bc36197..b69584193a 100644 --- a/depends/README.md +++ b/depends/README.md @@ -34,7 +34,7 @@ No other options are needed, the paths are automatically configured. #### For macOS cross compilation - sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python-setuptools + sudo apt-get install curl librsvg2-bin libtiff-tools bsdmainutils cmake imagemagick libcap-dev libz-dev libbz2-dev python3-setuptools #### For Win32/Win64 cross compilation diff --git a/doc/README.md b/doc/README.md index 51950d4a13..8876ffdd72 100644 --- a/doc/README.md +++ b/doc/README.md @@ -51,6 +51,7 @@ Development The Bitcoin repo's [root README](/README.md) contains relevant information on the development process and automated testing. - [Developer Notes](developer-notes.md) +- [Productivity Notes](productivity.md) - [Release Notes](release-notes.md) - [Release Process](release-process.md) - [Source Code Documentation (External Link)](https://dev.visucore.com/bitcoin/doxygen/) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index d21df36130..02a665008b 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -20,7 +20,8 @@ Supported API Given a transaction hash: returns a transaction in binary, hex-encoded binary, or JSON formats. -For full TX query capability, one must enable the transaction index via "txindex=1" command line / configuration option. +By default, this endpoint will only search the mempool. +To query for a confirmed transaction, enable the transaction index via "txindex=1" command line / configuration option. #### Blocks `GET /rest/block/<BLOCK-HASH>.<bin|hex|json>` diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index 70f5dfc882..d22b6e8383 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -1,6 +1,6 @@ FreeBSD build guide ====================== -(updated for FreeBSD 11.1) +(updated for FreeBSD 12.0) This guide describes how to build bitcoind and command-line utilities on FreeBSD. @@ -10,55 +10,51 @@ This guide does not contain instructions for building the GUI. You will need the following dependencies, which can be installed as root via pkg: -``` +```shell pkg install autoconf automake boost-libs git gmake libevent libtool openssl pkgconf + +git clone https://github.com/bitcoin/bitcoin.git ``` In order to run the test suite (recommended), you will need to have Python 3 installed: -``` +```shell pkg install python3 ``` -For the wallet (optional): -``` -./contrib/install_db4.sh `pwd` -export BDB_PREFIX="$PWD/db4" -``` - See [dependencies.md](dependencies.md) for a complete overview. -Download the source code: -``` -git clone https://github.com/bitcoin/bitcoin +### Building BerkeleyDB + +BerkeleyDB is only necessary for the wallet functionality. To skip this, pass +`--disable-wallet` to `./configure` and skip to the next section. + +```shell +./contrib/install_db4.sh `pwd` +export BDB_PREFIX="$PWD/db4" ``` ## Building Bitcoin Core **Important**: Use `gmake` (the non-GNU `make` will exit with an error): -``` +With wallet: +```shell ./autogen.sh - -./configure # to build with wallet OR -./configure --disable-wallet # to build without wallet +./configure --with-gui=no \ + BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ + BDB_CFLAGS="-I${BDB_PREFIX}/include" ``` -followed by either: - -``` -gmake +Without wallet: +```shell +./autogen.sh +./configure --with-gui=no --disable-wallet ``` -to build without testing, or +followed by: +```shell +gmake # use -jX here for parallelism +gmake check # Run tests if Python 3 is available ``` -gmake check -``` - -to also run the test suite (recommended, if Python 3 is installed). - -*Note on debugging*: The version of `gdb` installed by default is [ancient and considered harmful](https://wiki.freebsd.org/GdbRetirement). -It is not suitable for debugging a multi-threaded C++ program, not even for getting backtraces. Please install the package `gdb` and -use the versioned gdb command (e.g. `gdb7111`). - diff --git a/doc/build-unix.md b/doc/build-unix.md index 522e3069ce..da65bc347a 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -78,19 +78,13 @@ Now, you can either build from self-compiled [depends](/depends/README.md) or in BerkeleyDB is required for the wallet. -**For Ubuntu only:** db4.8 packages are available [here](https://launchpad.net/~bitcoin/+archive/bitcoin). -You can add the repository and install using the following commands: - - sudo apt-get install software-properties-common - sudo add-apt-repository ppa:bitcoin/bitcoin - sudo apt-get update - sudo apt-get install libdb4.8-dev libdb4.8++-dev - Ubuntu and Debian have their own libdb-dev and libdb++-dev packages, but these will install BerkeleyDB 5.1 or later. This will break binary wallet compatibility with the distributed executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass `--with-incompatible-bdb` to configure. +Otherwise, you can build from self-compiled `depends` (see above). + To build Bitcoin Core without wallet, see [*Disable-wallet mode*](/doc/build-unix.md#disable-wallet-mode) diff --git a/doc/build-windows.md b/doc/build-windows.md index 9641e0d3fd..036c585b44 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -71,6 +71,11 @@ If you want to build the windows installer with `make deploy` you need [NSIS](ht sudo apt install nsis +Acquire the source in the usual way: + + git clone https://github.com/bitcoin/bitcoin.git + cd bitcoin + ## Building for 64-bit Windows The first step is to install the mingw-w64 cross-compilation tool chain: @@ -87,11 +92,7 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that is located directly on the host Windows file system to perform the build. -Acquire the source in the usual way: - - git clone https://github.com/bitcoin/bitcoin.git - -Once the source code is ready the build steps are below: +Build using: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var cd depends @@ -115,11 +116,7 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that located directly on the host Windows file system to perform the build. -Acquire the source in the usual way: - - git clone https://github.com/bitcoin/bitcoin.git - -Then build using: +Build using: PATH=$(echo "$PATH" | sed -e 's/:\/mnt.*//g') # strip out problematic Windows %PATH% imported var cd depends diff --git a/doc/dependencies.md b/doc/dependencies.md index b833e9151f..0fb36184c2 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -17,14 +17,30 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | libevent | [2.1.8-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | | libjpeg | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L65) | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L64) | +| libsrvg | | | | | | | MiniUPnPc | [2.0.20180203](http://miniupnp.free.fr/files) | | No | | | | OpenSSL | [1.0.1k](https://www.openssl.org/source) | | Yes | | | | PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L66) | | protobuf | [2.6.1](https://github.com/google/protobuf/releases) | | No | | | -| Python (tests) | | [3.4](https://www.python.org/downloads) | | | | +| Python (tests) | | [3.5](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.2](https://github.com/bitcoin/bitcoin/pull/14725) | No | | | +| Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.5.1](https://github.com/bitcoin/bitcoin/issues/13478) | No | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L87) (Linux only) | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L86) (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | | zlib | [1.2.11](https://zlib.net/) | | | | No | + +Controlling dependencies +------------------------ +Some dependencies are not needed in all configurations. The following are some factors that affect the dependency list. + +#### Options passed to `./configure` +* MiniUPnPc is not needed with `--with-miniupnpc=no`. +* Berkeley DB is not needed with `--disable-wallet`. +* protobuf is not needed with `--disable-bip70`. +* Qt is not needed with `--without-gui`. +* If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. +* ZeroMQ is needed only with the `--with-zmq` option. + +#### Other +* librsvg is only needed if you need to run `make deploy` on (cross-compliation to) macOS.
\ No newline at end of file diff --git a/doc/descriptors.md b/doc/descriptors.md index de4d4e574f..5dbcd95e1d 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -39,7 +39,7 @@ Output descriptors currently support: ## Reference -Descriptors consist of several types of expressions. The top level expression is always a `SCRIPT`. +Descriptors consist of several types of expressions. The top level expression is either a `SCRIPT`, or `SCRIPT#CHECKSUM` where `CHECKSUM` is an 8-character alphanumeric descriptor checksum. `SCRIPT` expressions: - `sh(SCRIPT)` (top level only): P2SH embed the argument. @@ -169,3 +169,20 @@ existing Bitcoin Core wallets, a convenience function `combo` is provided, which takes as input a public key, and describes a set of P2PK, P2PKH, P2WPKH, and P2SH-P2WPH scripts for that key. In case the key is uncompressed, the set only includes P2PK and P2PKH scripts. + +### Checksums + +Descriptors can optionally be suffixed with a checksum to protect against +typos or copy-paste errors. + +These checksums consist of 8 alphanumeric characters. As long as errors are +restricted to substituting characters in `0123456789()[],'/*abcdefgh@:$%{}` +for others in that set and changes in letter case, up to 4 errors will always +be detected in descriptors up to 501 characters, and up to 3 errors in longer +ones. For larger numbers of errors, or other types of errors, there is a +roughly 1 in a trillion chance of not detecting the errors. + +All RPCs in Bitcoin Core will include the checksum in their output. Only +certain RPCs require checksums on input, including `deriveaddress` and +`importmulti`. The checksum for a descriptor without one can be computed +using the `getdescriptorinfo` RPC. diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 1deb5d791a..2662eea8fc 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -144,7 +144,7 @@ For example, to describe a function use: */ bool function(int arg1, const char *arg2) ``` -A complete list of `@xxx` commands can be found at http://www.stack.nl/~dimitri/doxygen/manual/commands.html. +A complete list of `@xxx` commands can be found at http://www.doxygen.nl/manual/commands.html. As Doxygen recognizes the comments by the delimiters (`/**` and `*/` in this case), you don't *need* to provide any commands for a comment to be valid; just a description text is fine. @@ -185,7 +185,7 @@ Not OK (used plenty in the current source, but not picked up): // ``` -A full list of comment syntaxes picked up by Doxygen can be found at https://www.stack.nl/~dimitri/doxygen/manual/docblocks.html, +A full list of comment syntaxes picked up by Doxygen can be found at http://www.doxygen.nl/manual/docblocks.html, but the above styles are favored. Documentation can be generated with `make docs` and cleaned up with `make clean-docs`. The resulting files are located in `doc/doxygen/html`; open `index.html` to view the homepage. @@ -844,54 +844,6 @@ would be to revert the upstream fix before applying the updates to Bitcoin's copy of LevelDB. In general you should be wary of any upstream changes affecting what data is returned from LevelDB queries. -Git and GitHub tips ---------------------- - -- For resolving merge/rebase conflicts, it can be useful to enable diff3 style using - `git config merge.conflictstyle diff3`. Instead of - - <<< - yours - === - theirs - >>> - - you will see - - <<< - yours - ||| - original - === - theirs - >>> - - This may make it much clearer what caused the conflict. In this style, you can often just look - at what changed between *original* and *theirs*, and mechanically apply that to *yours* (or the other way around). - -- When reviewing patches which change indentation in C++ files, use `git diff -w` and `git show -w`. This makes - the diff algorithm ignore whitespace changes. This feature is also available on github.com, by adding `?w=1` - at the end of any URL which shows a diff. - -- When reviewing patches that change symbol names in many places, use `git diff --word-diff`. This will instead - of showing the patch as deleted/added *lines*, show deleted/added *words*. - -- When reviewing patches that move code around, try using - `git diff --patience commit~:old/file.cpp commit:new/file/name.cpp`, and ignoring everything except the - moved body of code which should show up as neither `+` or `-` lines. In case it was not a pure move, this may - even work when combined with the `-w` or `--word-diff` options described above. - -- When looking at other's pull requests, it may make sense to add the following section to your `.git/config` - file: - - [remote "upstream-pull"] - fetch = +refs/pull/*:refs/remotes/upstream-pull/* - url = git@github.com:bitcoin/bitcoin.git - - This will add an `upstream-pull` remote to your git repository, which can be fetched using `git fetch --all` - or `git fetch upstream-pull`. Afterwards, you can use `upstream-pull/NUMBER/head` in arguments to `git show`, - `git checkout` and anywhere a commit id would be acceptable to see the changes from pull request NUMBER. - Scripted diffs -------------- diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 08b73d3b3c..f9221dde5b 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -5,6 +5,29 @@ A special test harness in `src/test/fuzz/` is provided for each fuzz target to provide an easy entry point for fuzzers and the like. In this document we'll describe how to use it with AFL and libFuzzer. +## Preparing fuzzing + +AFL needs an input directory with examples, and an output directory where it +will place examples that it found. These can be anywhere in the file system, +we'll define environment variables to make it easy to reference them. + +libFuzzer will use the input directory as output directory. + +Extract the example seeds (or other starting inputs) into the inputs +directory before starting fuzzing. + +``` +git clone https://github.com/bitcoin-core/qa-assets +export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus +``` + +Only for AFL: + +``` +mkdir outputs +export AFLOUT=$PWD/outputs +``` + ## AFL ### Building AFL @@ -23,7 +46,7 @@ export AFLPATH=$PWD To build Bitcoin Core using AFL instrumentation (this assumes that the `AFLPATH` was set as above): ``` -./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++ +./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++ export AFL_HARDEN=1 cd src/ make @@ -39,31 +62,14 @@ binary will be instrumented in such a way that the AFL features "persistent mode" and "deferred forkserver" can be used. See https://github.com/mcarpenter/afl/tree/master/llvm_mode for details. -### Preparing fuzzing - -AFL needs an input directory with examples, and an output directory where it -will place examples that it found. These can be anywhere in the file system, -we'll define environment variables to make it easy to reference them. - -``` -mkdir inputs -AFLIN=$PWD/inputs -mkdir outputs -AFLOUT=$PWD/outputs -``` - -Example inputs are available from: - -- https://download.visucore.com/bitcoin/bitcoin_fuzzy_in.tar.xz -- http://strateman.ninja/fuzzing.tar.xz - -Extract these (or other starting inputs) into the `inputs` directory before starting fuzzing. - ### Fuzzing To start the actual fuzzing use: + ``` -$AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/fuzz/fuzz_target_foo +export FUZZ_TARGET=fuzz_target_foo # Pick a fuzz_target +mkdir ${AFLOUT}/${FUZZ_TARGET} +$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- test/fuzz/${FUZZ_TARGET} ``` You may have to change a few kernel parameters to test optimally - `afl-fuzz` @@ -74,10 +80,10 @@ will print an error and suggestion if so. A recent version of `clang`, the address sanitizer and libFuzzer is needed (all found in the `compiler-rt` runtime libraries package). -To build the `test/test_bitcoin_fuzzy` executable run +To build all fuzz targets with libFuzzer, run ``` -./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++ +./configure --disable-ccache --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++ make ``` @@ -86,3 +92,6 @@ interchangeably between libFuzzer and AFL. See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer instrumented executable. + +Alternatively run the script in `./test/fuzz/test_runner.py` and provide it +with the `${DIR_FUZZ_IN}` created earlier. diff --git a/doc/man/bitcoin-cli.1 b/doc/man/bitcoin-cli.1 index 553addfa84..95c1d24dff 100644 --- a/doc/man/bitcoin-cli.1 +++ b/doc/man/bitcoin-cli.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.6. -.TH BITCOIN-CLI "1" "December 2018" "bitcoin-cli v0.17.99.0" "User Commands" +.TH BITCOIN-CLI "1" "February 2019" "bitcoin-cli v0.17.99.0" "User Commands" .SH NAME bitcoin-cli \- manual page for bitcoin-cli v0.17.99.0 .SH SYNOPSIS @@ -20,7 +20,7 @@ Bitcoin Core RPC client version v0.17.99.0 .HP \-? .IP -This help message +Print this help message and exit .HP \fB\-conf=\fR<file> .IP @@ -104,7 +104,7 @@ Chain selection options: .IP Use the test chain .SH COPYRIGHT -Copyright (C) 2009-2018 The Bitcoin Core developers +Copyright (C) 2009-2019 The Bitcoin Core developers Please contribute if you find Bitcoin Core useful. Visit <https://bitcoincore.org> for further information about the software. diff --git a/doc/man/bitcoin-qt.1 b/doc/man/bitcoin-qt.1 index 1d87acd3de..052d420608 100644 --- a/doc/man/bitcoin-qt.1 +++ b/doc/man/bitcoin-qt.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.6. -.TH BITCOIN-QT "1" "December 2018" "bitcoin-qt v0.17.99.0" "User Commands" +.TH BITCOIN-QT "1" "February 2019" "bitcoin-qt v0.17.99.0" "User Commands" .SH NAME bitcoin-qt \- manual page for bitcoin-qt v0.17.99.0 .SH SYNOPSIS @@ -56,7 +56,7 @@ Specify data directory .HP \fB\-dbcache=\fR<n> .IP -Set database cache size in megabytes (4 to 16384, default: 450) +Set database cache size in MiB (4 to 16384, default: 450) .HP \fB\-debuglogfile=\fR<file> .IP @@ -500,7 +500,7 @@ mining and transaction creation (default: 0.00001) \fB\-whitelistforcerelay\fR .IP Force relay of transactions from whitelisted peers even if they violate -local relay policy (default: 1) +local relay policy (default: 0) .HP \fB\-whitelistrelay\fR .IP @@ -606,7 +606,7 @@ Set SSL root certificates for payment request (default: \fB\-system\-\fR) .IP Show splash screen on startup (default: 1) .SH COPYRIGHT -Copyright (C) 2009-2018 The Bitcoin Core developers +Copyright (C) 2009-2019 The Bitcoin Core developers Please contribute if you find Bitcoin Core useful. Visit <https://bitcoincore.org> for further information about the software. diff --git a/doc/man/bitcoin-tx.1 b/doc/man/bitcoin-tx.1 index f16c68ca14..6b6071d9b7 100644 --- a/doc/man/bitcoin-tx.1 +++ b/doc/man/bitcoin-tx.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.6. -.TH BITCOIN-TX "1" "December 2018" "bitcoin-tx v0.17.99.0" "User Commands" +.TH BITCOIN-TX "1" "February 2019" "bitcoin-tx v0.17.99.0" "User Commands" .SH NAME bitcoin-tx \- manual page for bitcoin-tx v0.17.99.0 .SH SYNOPSIS @@ -14,7 +14,7 @@ Bitcoin Core bitcoin\-tx utility version v0.17.99.0 .HP \-? .IP -This help message +Print this help message and exit .HP \fB\-create\fR .IP @@ -92,7 +92,7 @@ sign=SIGHASH\-FLAGS .IP Add zero or more signatures to transaction. This command requires JSON registers:prevtxs=JSON object, privatekeys=JSON object. See -signrawtransaction docs for format of sighash flags, JSON +signrawtransactionwithkey docs for format of sighash flags, JSON objects. .PP Register Commands: @@ -105,7 +105,7 @@ set=NAME:JSON\-STRING .IP Set register NAME to given JSON\-STRING .SH COPYRIGHT -Copyright (C) 2009-2018 The Bitcoin Core developers +Copyright (C) 2009-2019 The Bitcoin Core developers Please contribute if you find Bitcoin Core useful. Visit <https://bitcoincore.org> for further information about the software. diff --git a/doc/man/bitcoin-wallet.1 b/doc/man/bitcoin-wallet.1 new file mode 100644 index 0000000000..1cb8cdebcd --- /dev/null +++ b/doc/man/bitcoin-wallet.1 @@ -0,0 +1,67 @@ +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.6. +.TH BITCOIN-WALLET "1" "February 2019" "bitcoin-wallet v0.17.99.0" "User Commands" +.SH NAME +bitcoin-wallet \- manual page for bitcoin-wallet v0.17.99.0 +.SH DESCRIPTION +Bitcoin Core bitcoin\-wallet version v0.17.99.0 +.PP +wallet\-tool is an offline tool for creating and interacting with Bitcoin Core wallet files. +By default wallet\-tool will act on wallets in the default mainnet wallet directory in the datadir. +To change the target wallet, use the \fB\-datadir\fR, \fB\-wallet\fR and \fB\-testnet\fR/\-regtest arguments. +.SS "Usage:" +.IP +bitcoin\-wallet [options] <command> +.SH OPTIONS +.HP +\-? +.IP +Print this help message and exit +.HP +\fB\-datadir=\fR<dir> +.IP +Specify data directory +.HP +\fB\-wallet=\fR<wallet\-name> +.IP +Specify wallet name +.PP +Debugging/Testing options: +.HP +\fB\-debug=\fR<category> +.IP +Output debugging information (default: 0). +.HP +\fB\-printtoconsole\fR +.IP +Send trace/debug info to console (default: 1 when no \fB\-debug\fR is true, 0 +otherwise. +.PP +Chain selection options: +.HP +\fB\-testnet\fR +.IP +Use the test chain +.PP +Commands: +.IP +create +.IP +Create new wallet file +.IP +info +.IP +Get wallet info +.SH COPYRIGHT +Copyright (C) 2009-2019 The Bitcoin Core developers + +Please contribute if you find Bitcoin Core useful. Visit +<https://bitcoincore.org> for further information about the software. +The source code is available from <https://github.com/bitcoin/bitcoin>. + +This is experimental software. +Distributed under the MIT software license, see the accompanying file COPYING +or <https://opensource.org/licenses/MIT> + +This product includes software developed by the OpenSSL Project for use in the +OpenSSL Toolkit <https://www.openssl.org> and cryptographic software written by +Eric Young and UPnP software written by Thomas Bernard. diff --git a/doc/man/bitcoind.1 b/doc/man/bitcoind.1 index 5c4b1cd03b..5e057d923f 100644 --- a/doc/man/bitcoind.1 +++ b/doc/man/bitcoind.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.6. -.TH BITCOIND "1" "December 2018" "bitcoind v0.17.99.0" "User Commands" +.TH BITCOIND "1" "February 2019" "bitcoind v0.17.99.0" "User Commands" .SH NAME bitcoind \- manual page for bitcoind v0.17.99.0 .SH SYNOPSIS @@ -56,7 +56,7 @@ Specify data directory .HP \fB\-dbcache=\fR<n> .IP -Set database cache size in megabytes (4 to 16384, default: 450) +Set database cache size in MiB (4 to 16384, default: 450) .HP \fB\-debuglogfile=\fR<file> .IP @@ -500,7 +500,7 @@ mining and transaction creation (default: 0.00001) \fB\-whitelistforcerelay\fR .IP Force relay of transactions from whitelisted peers even if they violate -local relay policy (default: 1) +local relay policy (default: 0) .HP \fB\-whitelistrelay\fR .IP @@ -580,7 +580,7 @@ Username for JSON\-RPC connections .IP Accept command line and JSON\-RPC commands .SH COPYRIGHT -Copyright (C) 2009-2018 The Bitcoin Core developers +Copyright (C) 2009-2019 The Bitcoin Core developers Please contribute if you find Bitcoin Core useful. Visit <https://bitcoincore.org> for further information about the software. diff --git a/doc/productivity.md b/doc/productivity.md new file mode 100644 index 0000000000..862017290d --- /dev/null +++ b/doc/productivity.md @@ -0,0 +1,161 @@ +Productivity Notes +================== + +Table of Contents +----------------- + +* [General](#general) + * [Cache compilations with `ccache`](#cache-compilations-with-ccache) + * [Disable features with `./configure`](#disable-features-with-configure) + * [Make use of your threads with `make -j`](#make-use-of-your-threads-with-make--j) + * [Multiple working directories with `git worktrees`](#multiple-working-directories-with-git-worktrees) +* [Writing code](#writing-code) + * [Format C/C++/Protobuf diffs with `clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy) + * [Format Python diffs with `yapf-diff.py`](#format-python-diffs-with-yapf-diffpy) +* [Rebasing/Merging code](#rebasingmerging-code) + * [More conflict context with `merge.conflictstyle diff3`](#more-conflict-context-with-mergeconflictstyle-diff3) +* [Reviewing code](#reviewing-code) + * [Reduce mental load with `git diff` options](#reduce-mental-load-with-git-diff-options) + * [Reference PRs easily with `refspec`s](#reference-prs-easily-with-refspecs) + * [Diff the diffs with `git range-diff`](#diff-the-diffs-with-git-range-diff) + +General +------ + +### Cache compilations with `ccache` + +The easiest way to faster compile times is to cache compiles. `ccache` is a way to do so, from its description at the time of writing: + +> ccache is a compiler cache. It speeds up recompilation by caching the result of previous compilations and detecting when the same compilation is being done again. Supported languages are C, C++, Objective-C and Objective-C++. + +Install `ccache` through your distribution's package manager, and run `./configure` with your normal flags to pick it up. + +To use ccache for all your C/C++ projects, follow the symlinks method [here](https://ccache.samba.org/manual/latest.html#_run_modes) to set it up. + +### Disable features with `./configure` + +After running `./autogen.sh`, which generates the `./configure` file, use `./configure --help` to identify features that you can disable to save on compilation time. A few common flags: + +```sh +--without-miniupnpc +--disable-bench +--disable-wallet +--without-gui +``` + +### Make use of your threads with `make -j` + +If you have multiple threads on your machine, you can tell `make` to utilize all of them with: + +```sh +make -j"$(($(nproc)+1))" +``` + +### Multiple working directories with `git worktrees` + +If you work with multiple branches or multiple copies of the repository, you should try `git worktrees`. + +To create a new branch that lives under a new working directory without disrupting your current working directory (useful for creating pull requests): +```sh +git worktree add -b my-shiny-new-branch ../living-at-my-new-working-directory based-on-my-crufty-old-commit-ish +``` + +To simply check out a commit-ish under a new working directory without disrupting your current working directory (useful for reviewing pull requests): +```sh +git worktree add --checkout ../where-my-checkout-commit-ish-will-live my-checkout-commit-ish +``` + +----- + +This synergizes well with [`ccache`](#cache-compilations-with-ccache) as objects resulting from unchanged code will most likely hit the cache and won't need to be recompiled. + +You can also set up [upstream refspecs](#reference-prs-easily-with-refspecs) to refer to pull requests easier in the above `git worktree` commands. + +Writing code +------------ + +### Format C/C++/Protobuf diffs with `clang-format-diff.py` + +See [contrib/devtools/README.md](contrib/devtools/README.md#clang-format-diff.py). + +### Format Python diffs with `yapf-diff.py` + +Usage is exactly the same as [`clang-format-diff.py`](#format-ccprotobuf-diffs-with-clang-format-diffpy). You can get it [here](https://github.com/MarcoFalke/yapf-diff). + +Rebasing/Merging code +------------- + +### More conflict context with `merge.conflictstyle diff3` + +For resolving merge/rebase conflicts, it can be useful to enable diff3 style using `git config merge.conflictstyle diff3`. Instead of + +```diff +<<< +yours +=== +theirs +>>> +``` + + you will see + +```diff +<<< +yours +||| +original +=== +theirs +>>> +``` + +This may make it much clearer what caused the conflict. In this style, you can often just look at what changed between *original* and *theirs*, and mechanically apply that to *yours* (or the other way around). + +Reviewing code +-------------- + +### Reduce mental load with `git diff` options + +When reviewing patches which change indentation in C++ files, use `git diff -w` and `git show -w`. This makes the diff algorithm ignore whitespace changes. This feature is also available on github.com, by adding `?w=1` at the end of any URL which shows a diff. + +When reviewing patches that change symbol names in many places, use `git diff --word-diff`. This will instead of showing the patch as deleted/added *lines*, show deleted/added *words*. + +When reviewing patches that move code around, try using `git diff --patience commit~:old/file.cpp commit:new/file/name.cpp`, and ignoring everything except the moved body of code which should show up as neither `+` or `-` lines. In case it was not a pure move, this may even work when combined with the `-w` or `--word-diff` options described above. `--color-moved=dimmed-zebra` will also dim the coloring of moved hunks in the diff on compatible terminals. + +### Reference PRs easily with `refspec`s + +When looking at other's pull requests, it may make sense to add the following section to your `.git/config` file: + +``` +[remote "upstream-pull"] + fetch = +refs/pull/*:refs/remotes/upstream-pull/* + url = git@github.com:bitcoin/bitcoin.git +``` + +This will add an `upstream-pull` remote to your git repository, which can be fetched using `git fetch --all` or `git fetch upstream-pull`. Afterwards, you can use `upstream-pull/NUMBER/head` in arguments to `git show`, `git checkout` and anywhere a commit id would be acceptable to see the changes from pull request NUMBER. + +### Diff the diffs with `git range-diff` + +It is very common for contributors to rebase their pull requests, or make changes to commits (perhaps in response to review) that are not at the head of their branch. This poses a problem for reviewers as when the contributor force pushes, the reviewer is no longer sure that his previous reviews of commits are still valid (as the commit hashes can now be different even though the diff is semantically the same). `git range-diff` can help solve this problem by diffing the diffs. + +For example, to identify the differences between your previously reviewed diffs P1-5, and the new diffs P1-2,N3-4 as illustrated below: +``` + P1--P2--P3--P4--P5 <-- previously-reviewed-head + / +...--m <-- master + \ + P1--P2--N3--N4--N5 <-- new-head (with P3 slightly modified) +``` + +You can do: +```sh +git range-diff master previously-reviewed-head new-head +``` + +Note that `git range-diff` also work for rebases. + +----- + +`git range-diff` also accepts normal `git diff` options, see [Reduce mental load with `git diff` options](#reduce-mental-load-with-git-diff-options) for useful `git diff` options. + +You can also set up [upstream refspecs](#reference-prs-easily-with-refspecs) to refer to pull requests easier in the above `git range-diff` commands. diff --git a/doc/release-notes-14941.md b/doc/release-notes-14941.md deleted file mode 100644 index c3820d0368..0000000000 --- a/doc/release-notes-14941.md +++ /dev/null @@ -1,5 +0,0 @@ -Miscellaneous RPC changes ------------- - -- The `unloadwallet` RPC is now synchronous, meaning that it blocks until the - wallet is fully unloaded. diff --git a/doc/release-notes-14982.md b/doc/release-notes-14982.md deleted file mode 100644 index 3f0bf8aacd..0000000000 --- a/doc/release-notes-14982.md +++ /dev/null @@ -1,5 +0,0 @@ -New RPCs --------- - -- The RPC `getrpcinfo` returns runtime details of the RPC server. At the moment - it returns the active commands and the corresponding execution time. diff --git a/doc/release-notes-15566.md b/doc/release-notes-15566.md new file mode 100644 index 0000000000..49964d7550 --- /dev/null +++ b/doc/release-notes-15566.md @@ -0,0 +1,3 @@ +Miscellaneous CLI Changes +------------------------- +- The `testnet` field in `bitcoin-cli -getinfo` has been renamed to `chain` and now returns the current network name as defined in BIP70 (main, test, regtest).
\ No newline at end of file diff --git a/doc/release-notes.md b/doc/release-notes.md index c23a7f6e0a..ebcdcda306 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -66,255 +66,15 @@ platform. Notable changes =============== -Mining ------- - -- Calls to `getblocktemplate` will fail if the segwit rule is not specified. - Calling `getblocktemplate` without segwit specified is almost certainly - a misconfiguration since doing so results in lower rewards for the miner. - Failed calls will produce an error message describing how to enable the - segwit rule. - -Configuration option changes ----------------------------- - -- A warning is printed if an unrecognized section name is used in the - configuration file. Recognized sections are `[test]`, `[main]`, and - `[regtest]`. - -- Four new options are available for configuring the maximum number of - messages that ZMQ will queue in memory (the "high water mark") before - dropping additional messages. The default value is 1,000, the same as - was used for previous releases. See the [ZMQ - documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/zmq.md#usage) - for details. - -- The `enablebip61` option (introduced in Bitcoin Core 0.17.0) is - used to toggle sending of BIP 61 reject messages. Reject messages have no use - case on the P2P network and are only logged for debugging by most network - nodes. The option will now by default be off for improved privacy and security - as well as reduced upload usage. The option can explicitly be turned on for - local-network debugging purposes. - -- The `rpcallowip` option can no longer be used to automatically listen - on all network interfaces. Instead, the `rpcbind` parameter must also - be used to specify the IP addresses to listen on. Listening for RPC - commands over a public network connection is insecure and should be - disabled, so a warning is now printed if a user selects such a - configuration. If you need to expose RPC in order to use a tool - like Docker, ensure you only bind RPC to your localhost, e.g. `docker - run [...] -p 127.0.0.1:8332:8332` (this is an extra `:8332` over the - normal Docker port specification). - -- The `rpcpassword` option now causes a startup error if the password - set in the configuration file contains a hash character (#), as it's - ambiguous whether the hash character is meant for the password or as a - comment. - -- The `whitelistforcerelay` option is used to relay transactions from - whitelisted peers even when not accepted to the mempool. This option now - defaults to being off, so that changes in policy and disconnect/ban behavior - will not cause a node that is whitelisting another to be dropped by peers. - Users can still explicitly enable this behavior with the command line option - (and may want to consider letting the Bitcoin Core project know about their - use-case, as this feature could be deprecated in the future). - -Documentation -------------- - -- A new short - [document](https://github.com/bitcoin/bitcoin/blob/master/doc/JSON-RPC-interface.md) - about the JSON-RPC interface describes cases where the results of an - RPC might contain inconsistencies between data sourced from different - subsystems, such as wallet state and mempool state. A note is added - to the [REST interface documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/REST-interface.md) - indicating that the same rules apply. - -- A new [document](https://github.com/bitcoin/bitcoin/blob/master/doc/bitcoin-conf.md) - about the `bitcoin.conf` file describes how to use it to configure - Bitcoin Core. - -- A new document introduces Bitcoin Core's BIP174 - [Partially-Signed Bitcoin Transactions (PSBT)](https://github.com/bitcoin/bitcoin/blob/master/doc/psbt.md) - interface, which is used to allow multiple programs to collaboratively - work to create, sign, and broadcast new transactions. This is useful - for offline (cold storage) wallets, multisig wallets, coinjoin - implementations, and many other cases where two or more programs need - to interact to generate a complete transaction. - -- The [output script descriptor](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md) - documentation has been updated with information about new features in - this still-developing language for describing the output scripts that - a wallet or other program wants to receive notifications for, such as - which addresses it wants to know received payments. The language is - currently used in the `scantxoutset` RPC and is expected to be adapted - to other RPCs and to the underlying wallet structure. - -Build system changes --------------------- - -- A new `--disable-bip70` option may be passed to `./configure` to - prevent Bitcoin-Qt from being built with support for the BIP70 payment - protocol or from linking libssl. As the payment protocol has exposed - Bitcoin Core to libssl vulnerabilities in the past, builders who don't - need BIP70 support are encouraged to use this option to reduce their - exposure to future vulnerabilities. - -Deprecated or removed RPCs --------------------------- - -- The `signrawtransaction` RPC is removed after being deprecated and - hidden behind a special configuration option in version 0.17.0. - -- The 'account' API is removed after being deprecated in v0.17. The - 'label' API was introduced in v0.17 as a replacement for accounts. - See the [release notes from v0.17](https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.17.0.md#label-and-account-apis-for-wallet) - for a full description of the changes from the 'account' API to the - 'label' API. - -- The `addwitnessaddress` RPC is removed after being deprecated in - version 0.13.0. - -- The wallet's `generate` RPC method is deprecated and will be fully - removed in a subsequent major version. This RPC is only used for - testing, but its implementation reached across multiple subsystems - (wallet and mining), so it is being deprecated to simplify the - wallet-node interface. Projects that are using `generate` for testing - purposes should transition to using the `generatetoaddress` RPC, which - does not require or use the wallet component. Calling - `generatetoaddress` with an address returned by the `getnewaddress` - RPC gives the same functionality as the old `generate` RPC. To - continue using `generate` in this version, restart bitcoind with the - `-deprecatedrpc=generate` configuration option. - -New RPCs --------- - -- A new `getnodeaddresses` RPC returns peer addresses known to this - node. It may be used to find nodes to connect to without using a DNS - seeder. - -- A new `listwalletdir` RPC returns a list of wallets in the wallet - directory (either the default wallet directory or the directory - configured by the `-walletdir` parameter). - -Updated RPCs +Example item ------------ -Note: some low-level RPC changes mainly useful for testing are described -in the Low-level Changes section below. - -- The `getpeerinfo` RPC now returns an additional `minfeefilter` field - set to the peer's BIP133 fee filter. You can use this to detect that - you have peers that are willing to accept transactions below the - default minimum relay fee. - -- The mempool RPCs, such as `getrawmempool` with `verbose=true`, now - return an additional "bip125-replaceable" value indicating whether the - transaction (or its unconfirmed ancestors) opts-in to asking nodes and - miners to replace it with a higher-feerate transaction spending any of - the same inputs. - -- The `settxfee` RPC previously silently ignored attempts to set the fee - below the allowed minimums. It now prints a warning. The special - value of "0" may still be used to request the minimum value. - -- The `getaddressinfo` RPC now provides an `ischange` field indicating - whether the wallet used the address in a change output. - -- The `importmulti` RPC has been updated to support P2WSH, P2WPKH, - P2SH-P2WPKH, and P2SH-P2WSH. Requests for P2WSH and P2SH-P2WSH accept - an additional `witnessscript` parameter. - -- The `importmulti` RPC now returns an additional `warnings` field for - each request with an array of strings explaining when fields are being - ignored or are inconsistent, if there are any. - -- The `getaddressinfo` RPC now returns an additional `solvable` boolean - field when Bitcoin Core knows enough about the address's scriptPubKey, - optional redeemScript, and optional witnessScript in order for the - wallet to be able to generate an unsigned input spending funds sent to - that address. - -- The `getaddressinfo`, `listunspent`, and `scantxoutset` RPCs now - return an additional `desc` field that contains an output descriptor - containing all key paths and signing information for the address - (except for the private key). The `desc` field is only returned for - `getaddressinfo` and `listunspent` when the address is solvable. - -- The `importprivkey` RPC will preserve previously-set labels for - addresses or public keys corresponding to the private key being - imported. For example, if you imported a watch-only address with the - label "cold wallet" in earlier releases of Bitcoin Core, subsequently - importing the private key would default to resetting the address's - label to the default empty-string label (""). In this release, the - previous label of "cold wallet" will be retained. If you optionally - specify any label besides the default when calling `importprivkey`, - the new label will be applied to the address. - -- See the [Mining](#mining) section for changes to `getblocktemplate`. - -- The `getrawtransaction` RPC & REST endpoints no longer check the - unspent UTXO set for a transaction. The remaining behaviors are as - follows: 1. If a blockhash is provided, check the corresponding block. - 2. If no blockhash is provided, check the mempool. 3. If no blockhash - is provided but txindex is enabled, also check txindex. - -Graphical User Interface (GUI) ------------------------------- - -- A new Window menu is added alongside the existing File, Settings, and - Help menus. Several items from the other menus that opened new - windows have been moved to this new Window menu. - -- In the Send tab, the checkbox for "pay only the required fee" - has been removed. Instead, the user can simply decrease the value in - the Custom Feerate field all the way down to the node's configured - minimum relay fee. - -- In the Overview tab, the watch-only balance will be the only - balance shown if the wallet was created using the `createwallet` RPC - and the `disable_private_keys` parameter was set to true. - -- The launch-on-startup option is no longer available on macOS if - compiled with macosx min version greater than 10.11 (use - CXXFLAGS="-mmacosx-version-min=10.11" - CFLAGS="-mmacosx-version-min=10.11" for setting the deployment - sdk version) Low-level changes ================= -RPC ---- - -- The `submitblock` RPC previously returned the reason a rejected block - was invalid the first time it processed that block but returned a - generic "duplicate" rejection message on subsequent occasions it - processed the same block. It now always returns the fundamental - reason for rejecting an invalid block and only returns "duplicate" for - valid blocks it has already accepted. - -- A new `submitheader` RPC allows submitting block headers independently - from their block. This is likely only useful for testing. - -Configuration -------------- - -- The `-usehd` configuration option was removed in version 0.16. From - that version onwards, all new wallets created are hierarchical - deterministic wallets. This release makes specifying `-usehd` an - invalid configuration option. - -Changes for particular platforms --------------------------------- - -- On macOS, Bitcoin Core now opts out of application CPU throttling - ("app nap") during initial blockchain download, when catching up from - over 100 blocks behind the current chain tip, or when reindexing chain - data. This helps prevent these operations from taking an excessively - long time because the operating system is attempting to conserve - power. +Example item +------------ Credits ======= @@ -322,4 +82,4 @@ Credits Thanks to everyone who directly contributed to this release: -As well as everyone that helped translating on [Transifex](https://www.transifex.com/projects/p/bitcoin/). +As well as everyone that helped translating on [Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-pr12255.md b/doc/release-notes/release-notes-pr12255.md deleted file mode 100644 index 5ac8b44283..0000000000 --- a/doc/release-notes/release-notes-pr12255.md +++ /dev/null @@ -1,17 +0,0 @@ -systemd init file -========= - -The systemd init file (`contrib/init/bitcoind.service`) has been changed to use -`/var/lib/bitcoind` as the data directory instead of `~bitcoin/.bitcoin`. This -change makes Bitcoin Core more consistent with other services, and makes the -systemd init config more consistent with existing Upstart and OpenRC configs. - -The configuration, PID, and data directories are now completely managed by -systemd, which will take care of their creation, permissions, etc. See -[`systemd.exec (5)`](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RuntimeDirectory=) -for more details. - -When using the provided init files under `contrib/init`, overriding the -`datadir` option in `/etc/bitcoin/bitcoin.conf` will have no effect. This is -because the command line arguments specified in the init files take precedence -over the options specified in `/etc/bitcoin/bitcoin.conf`. diff --git a/doc/release-process.md b/doc/release-process.md index d20a3dc6b3..eb1f5ad222 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -291,24 +291,48 @@ bitcoin.org (see below for bitcoin.org update instructions). - After the pull request is merged, the website will automatically show the newest version within 15 minutes, as well as update the OS download links. Ping @saivann/@harding (saivann/harding on Freenode) in case anything goes wrong -- Announce the release: - - - bitcoin-dev and bitcoin-core-dev mailing list - - - Bitcoin Core announcements list https://bitcoincore.org/en/list/announcements/join/ +- Update other repositories and websites for new version - bitcoincore.org blog post - bitcoincore.org RPC documentation update - - Update title of #bitcoin on Freenode IRC + - Update packaging repo - - Optionally twitter, reddit /r/Bitcoin, ... but this will usually sort out itself + - Notify BlueMatt so that he can start building [the PPAs](https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin) + + - Create a new branch for the major release "0.xx" (used to build the snap package) + + - Notify MarcoFalke so that he can start building the snap package + + - https://code.launchpad.net/~bitcoin-core/bitcoin-core-snap/+git/packaging (Click "Import Now" to fetch the branch) + - https://code.launchpad.net/~bitcoin-core/bitcoin-core-snap/+git/packaging/+ref/0.xx (Click "Create snap package") + - Name it "bitcoin-core-snap-0.xx" + - Leave owner and series as-is + - Select architectures that are compiled via gitian + - Leave "automatically build when branch changes" unticked + - Tick "automatically upload to store" + - Put "bitcoin-core" in the registered store package name field + - Tick the "edge" box + - Put "0.xx" in the track field + - Click "create snap package" + - Click "Request builds" for every new release on this branch (after updating the snapcraft.yml in the branch to reflect the latest gitian results) + - Promote release on https://snapcraft.io/bitcoin-core/releases if it passes sanity checks - - Notify BlueMatt so that he can start building [the PPAs](https://launchpad.net/~bitcoin/+archive/ubuntu/bitcoin) + - This repo - - Archive release notes for the new version to `doc/release-notes/` (branch `master` and branch of the release) + - Archive release notes for the new version to `doc/release-notes/` (branch `master` and branch of the release) - - Create a [new GitHub release](https://github.com/bitcoin/bitcoin/releases/new) with a link to the archived release notes. + - Create a [new GitHub release](https://github.com/bitcoin/bitcoin/releases/new) with a link to the archived release notes. + +- Announce the release: + + - bitcoin-dev and bitcoin-core-dev mailing list + + - Bitcoin Core announcements list https://bitcoincore.org/en/list/announcements/join/ + + - Update title of #bitcoin on Freenode IRC + + - Optionally twitter, reddit /r/Bitcoin, ... but this will usually sort out itself - Celebrate diff --git a/doc/translation_process.md b/doc/translation_process.md index 9692832842..b9a10b6527 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -8,7 +8,7 @@ Transifex is setup to monitor the GitHub repo for updates, and when code contain Multiple language support is critical in assisting Bitcoin’s global adoption, and growth. One of Bitcoin’s greatest strengths is cross-border money transfers, any help making that easier is greatly appreciated. -See the [Transifex Bitcoin project](https://www.transifex.com/projects/p/bitcoin/) to assist in translations. You should also join the translation mailing list for announcements - see details below. +See the [Transifex Bitcoin project](https://www.transifex.com/bitcoin/bitcoin/) to assist in translations. You should also join the translation mailing list for announcements - see details below. ### Writing code with translations We use automated scripts to help extract translations in both Qt, and non-Qt source files. It is rarely necessary to manually edit the files in `src/qt/locale/`. The translation source files must adhere to the following format: @@ -43,7 +43,7 @@ git commit ### Creating a Transifex account Visit the [Transifex Signup](https://www.transifex.com/signup/) page to create an account. Take note of your username and password, as they will be required to configure the command-line tool. -You can find the Bitcoin translation project at [https://www.transifex.com/projects/p/bitcoin/](https://www.transifex.com/projects/p/bitcoin/). +You can find the Bitcoin translation project at [https://www.transifex.com/bitcoin/bitcoin/](https://www.transifex.com/bitcoin/bitcoin/). ### Installing the Transifex client command-line tool The client is used to fetch updated translations. If you are having problems, or need more details, see [https://docs.transifex.com/client/installing-the-client](https://docs.transifex.com/client/installing-the-client) diff --git a/doc/translation_strings_policy.md b/doc/translation_strings_policy.md index 737d11f045..634aca3559 100644 --- a/doc/translation_strings_policy.md +++ b/doc/translation_strings_policy.md @@ -33,25 +33,25 @@ General recommendations Try not to burden translators with translating messages that are e.g. slight variations of other messages. In the GUI, avoid the use of text where an icon or symbol will do. -Make sure that placeholder texts in forms don't end up in the list of strings to be translated (use `<string notr="true">`). +Make sure that placeholder texts in forms do not end up in the list of strings to be translated (use `<string notr="true">`). ### Make translated strings understandable -Try to write translation strings in an understandable way, for both the user and the translator. Avoid overly technical or detailed messages +Try to write translation strings in an understandable way, for both the user and the translator. Avoid overly technical or detailed messages. ### Do not translate internal errors -Do not translate internal errors, or log messages, or messages that appear on the RPC interface. If an error is to be shown to the user, -use a translatable generic message, then log the detailed message to the log. E.g. "A fatal internal error occurred, see debug.log for details". +Do not translate internal errors, log messages, or messages that appear on the RPC interface. If an error is to be shown to the user, +use a translatable generic message, then log the detailed message to the log. E.g., "A fatal internal error occurred, see debug.log for details". This helps troubleshooting; if the error is the same for everyone, the likelihood is increased that it can be found using a search engine. ### Avoid fragments -Avoid dividing up a message into fragments. Translators see every string separately, so may misunderstand the context if the messages are not self-contained. +Avoid dividing up a message into fragments. Translators see every string separately, so they may misunderstand the context if the messages are not self-contained. ### Avoid HTML in translation strings -There have been difficulties with use of HTML in translation strings; translators should not be able to accidentally affect the formatting of messages. +There have been difficulties with the use of HTML in translation strings; translators should not be able to accidentally affect the formatting of messages. This may sometimes be at conflict with the recommendation in the previous section. ### Plurals @@ -66,7 +66,7 @@ Plurals can be complex in some languages. A quote from the gettext documentation 25-31 pliko'w and so on -In Qt code use tr's third argument for optional plurality. For example: +In Qt code, use tr's third argument for optional plurality. For example: tr("%n hour(s)","",secs/HOUR_IN_SECONDS); tr("%n day(s)","",secs/DAY_IN_SECONDS); @@ -82,7 +82,7 @@ This adds `<numerusform>`s to the respective `.ts` file, which can be translated </translation> </message> -Where it is possible try to avoid embedding numbers into the flow of the string at all. e.g. +Where possible, try to avoid embedding numbers into the flow of the string at all. E.g., WARNING: check your network connection, %d blocks received in the last %d hours (%d expected) diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index 6062d49c40..b5475dc1c6 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -4,6 +4,10 @@ # Network-related settings: +# Note that if you use testnet or regtest, particularly with the options +# addnode, connect, port, bind, rpcport, rpcbind or wallet, you will also +# want to read "[Sections]" further down. + # Run on the test network instead of the real bitcoin network. #testnet=0 @@ -53,6 +57,9 @@ # Listening mode, enabled by default except when 'connect' is being used #listen=1 +# Port on which to listen for connections (default: 8333, testnet: 18333, regtest: 18444) +#port= + # Maximum number of inbound+outbound connections. #maxconnections= @@ -115,6 +122,10 @@ # Wallet options +# Specify where to find wallet, lockfile and logs. If not present, those files will be +# created as new. +#wallet=</path/to/dir> + # Create transactions that have enough fees so they are likely to begin confirmation within n blocks (default: 6). # This setting is over-ridden by the -paytxfee option. #txconfirmtarget=n @@ -142,3 +153,19 @@ # Minimize to the system tray #minimizetotray=1 + +# [Sections] +# Most options apply to mainnet, testnet and regtest. +# If you want to confine an option to just one network, you should add it in the +# relevant section below. +# EXCEPTIONS: The options addnode, connect, port, bind, rpcport, rpcbind and wallet +# only apply to mainnet unless they appear in the appropriate section below. + +# Options only for mainnet +[main] + +# Options only for testnet +[test] + +# Options only for regtest +[regtest] diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 148cddf02d..acf9e86667 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -80,6 +80,7 @@ Section -Main SEC0000 File @abs_top_srcdir@/release/@BITCOIN_DAEMON_NAME@@EXEEXT@ File @abs_top_srcdir@/release/@BITCOIN_CLI_NAME@@EXEEXT@ File @abs_top_srcdir@/release/@BITCOIN_TX_NAME@@EXEEXT@ + File @abs_top_srcdir@/release/@BITCOIN_WALLET_TOOL_NAME@@EXEEXT@ SetOutPath $INSTDIR\doc File /r /x Makefile* @abs_top_srcdir@/doc\*.* SetOutPath $INSTDIR diff --git a/src/Makefile.am b/src/Makefile.am index a57fcb0711..14db963253 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -118,6 +118,7 @@ BITCOIN_CORE_H = \ clientversion.h \ coins.h \ compat.h \ + compat/assumptions.h \ compat/byteswap.h \ compat/endian.h \ compat/sanity.h \ @@ -127,6 +128,7 @@ BITCOIN_CORE_H = \ core_io.h \ core_memusage.h \ cuckoocache.h \ + flatfile.h \ fs.h \ httprpc.h \ httpserver.h \ @@ -152,6 +154,7 @@ BITCOIN_CORE_H = \ netaddress.h \ netbase.h \ netmessagemaker.h \ + node/transaction.h \ noui.h \ optional.h \ outputtype.h \ @@ -161,6 +164,7 @@ BITCOIN_CORE_H = \ policy/rbf.h \ pow.h \ protocol.h \ + psbt.h \ random.h \ reverse_iterator.h \ reverselock.h \ @@ -194,6 +198,7 @@ BITCOIN_CORE_H = \ txmempool.h \ ui_interface.h \ undo.h \ + util/bip32.h \ util/bytevectorhash.h \ util/system.h \ util/memory.h \ @@ -209,6 +214,7 @@ BITCOIN_CORE_H = \ wallet/db.h \ wallet/feebumper.h \ wallet/fees.h \ + wallet/psbtwallet.h \ wallet/rpcwallet.h \ wallet/wallet.h \ wallet/walletdb.h \ @@ -242,6 +248,7 @@ libbitcoin_server_a_SOURCES = \ chain.cpp \ checkpoints.cpp \ consensus/tx_verify.cpp \ + flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ index/base.cpp \ @@ -255,6 +262,7 @@ libbitcoin_server_a_SOURCES = \ miner.cpp \ net.cpp \ net_processing.cpp \ + node/transaction.cpp \ noui.cpp \ outputtype.cpp \ policy/fees.cpp \ @@ -308,6 +316,7 @@ libbitcoin_wallet_a_SOURCES = \ wallet/feebumper.cpp \ wallet/fees.cpp \ wallet/init.cpp \ + wallet/psbtwallet.cpp \ wallet/rpcdump.cpp \ wallet/rpcwallet.cpp \ wallet/wallet.cpp \ @@ -421,6 +430,7 @@ libbitcoin_common_a_SOURCES = \ netaddress.cpp \ netbase.cpp \ policy/feerate.cpp \ + psbt.cpp \ protocol.cpp \ scheduler.cpp \ script/descriptor.cpp \ @@ -450,6 +460,7 @@ libbitcoin_util_a_SOURCES = \ support/cleanse.cpp \ sync.cpp \ threadinterrupt.cpp \ + util/bip32.cpp \ util/bytevectorhash.cpp \ util/system.cpp \ util/moneystr.cpp \ @@ -557,12 +568,14 @@ bitcoin_wallet_LDADD = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) \ + $(LIBBITCOIN_ZMQ) \ $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ - $(LIBSECP256K1) + $(LIBSECP256K1) \ + $(LIBUNIVALUE) -bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) +bitcoin_wallet_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(ZMQ_LIBS) # # bitcoinconsensus library # diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 5e787ca222..3042f6df19 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -26,6 +26,7 @@ bench_bench_bitcoin_SOURCES = \ bench/gcs_filter.cpp \ bench/merkle_root.cpp \ bench/mempool_eviction.cpp \ + bench/rpc_mempool.cpp \ bench/verify_script.cpp \ bench/base58.cpp \ bench/bech32.cpp \ @@ -37,6 +38,7 @@ nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) bench_bench_bitcoin_LDADD = \ + $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_COMMON) \ @@ -47,7 +49,9 @@ bench_bench_bitcoin_LDADD = \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ - $(LIBUNIVALUE) + $(LIBUNIVALUE) \ + $(EVENT_PTHREADS_LIBS) \ + $(EVENT_LIBS) if ENABLE_ZMQ bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) @@ -57,7 +61,7 @@ if ENABLE_WALLET bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp endif -bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) +bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(CRYPTO_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 12112375ea..a6e0785616 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -2,7 +2,6 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -bin_PROGRAMS += test/test_bitcoin FUZZ_TARGETS = \ test/fuzz/address_deserialize \ @@ -21,6 +20,7 @@ FUZZ_TARGETS = \ test/fuzz/inv_deserialize \ test/fuzz/messageheader_deserialize \ test/fuzz/netaddr_deserialize \ + test/fuzz/script_flags \ test/fuzz/service_deserialize \ test/fuzz/transaction_deserialize \ test/fuzz/txoutcompressor_deserialize \ @@ -28,6 +28,8 @@ FUZZ_TARGETS = \ if ENABLE_FUZZ noinst_PROGRAMS += $(FUZZ_TARGETS:=) +else +bin_PROGRAMS += test/test_bitcoin endif TEST_SRCDIR = test @@ -49,14 +51,32 @@ RAW_TEST_FILES = GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) BITCOIN_TEST_SUITE = \ - test/test_bitcoin_main.cpp \ + test/main.cpp \ test/test_bitcoin.h \ test/test_bitcoin.cpp FUZZ_SUITE = \ + test/test_bitcoin.h \ + test/test_bitcoin.cpp \ test/fuzz/fuzz.cpp \ test/fuzz/fuzz.h +FUZZ_SUITE_LD_COMMON = \ + $(LIBBITCOIN_SERVER) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBUNIVALUE) \ + $(LIBLEVELDB) \ + $(LIBLEVELDB_SSE42) \ + $(BOOST_LIBS) \ + $(LIBMEMENV) \ + $(LIBSECP256K1) \ + $(EVENT_LIBS) \ + $(CRYPTO_LIBS) \ + $(EVENT_PTHREADS_LIBS) + # test_bitcoin binary # BITCOIN_TESTS =\ test/arith_uint256_tests.cpp \ @@ -81,6 +101,7 @@ BITCOIN_TESTS =\ test/cuckoocache_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ + test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ @@ -88,7 +109,7 @@ BITCOIN_TESTS =\ test/key_tests.cpp \ test/limitedmap_tests.cpp \ test/dbwrapper_tests.cpp \ - test/main_tests.cpp \ + test/validation_tests.cpp \ test/mempool_tests.cpp \ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ @@ -167,349 +188,135 @@ test_test_bitcoin_LDADD += $(BDB_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(RAPIDC test_test_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static if ENABLE_ZMQ -test_test_bitcoin_LDADD += $(ZMQ_LIBS) +test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif if ENABLE_FUZZ -test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_block_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_block_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCK_DESERIALIZE=1 test_fuzz_block_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_block_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_block_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_block_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_block_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_transaction_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_transaction_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTRANSACTION_DESERIALIZE=1 test_fuzz_transaction_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_transaction_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_transaction_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blocklocator_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocklocator_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKLOCATOR_DESERIALIZE=1 test_fuzz_blocklocator_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocklocator_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocklocator_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blocklocator_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blocklocator_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blockmerkleroot_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockmerkleroot_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKMERKLEROOT=1 test_fuzz_blockmerkleroot_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockmerkleroot_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockmerkleroot_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blockmerkleroot_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blockmerkleroot_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_addrman_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_addrman_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRMAN_DESERIALIZE=1 test_fuzz_addrman_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addrman_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_addrman_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_addrman_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_addrman_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blockheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKHEADER_DESERIALIZE=1 test_fuzz_blockheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockheader_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blockheader_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blockheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_banentry_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_banentry_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBANENTRY_DESERIALIZE=1 test_fuzz_banentry_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_banentry_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_banentry_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_banentry_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_banentry_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_txundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXUNDO_DESERIALIZE=1 test_fuzz_txundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_txundo_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_txundo_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_txundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blockundo_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blockundo_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKUNDO_DESERIALIZE=1 test_fuzz_blockundo_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blockundo_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blockundo_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blockundo_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blockundo_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_coins_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_coins_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_coins_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_netaddr_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_netaddr_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DNETADDR_DESERIALIZE=1 test_fuzz_netaddr_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_netaddr_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_netaddr_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_netaddr_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_netaddr_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_script_flags_SOURCES = $(FUZZ_SUITE) test/fuzz/script_flags.cpp +test_fuzz_script_flags_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_script_flags_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_script_flags_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_script_flags_LDADD = $(FUZZ_SUITE_LD_COMMON) + +test_fuzz_service_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_service_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DSERVICE_DESERIALIZE=1 test_fuzz_service_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_service_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_service_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_service_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_service_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_messageheader_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_messageheader_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DMESSAGEHEADER_DESERIALIZE=1 test_fuzz_messageheader_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_messageheader_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_messageheader_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_messageheader_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_messageheader_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_address_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_address_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDRESS_DESERIALIZE=1 test_fuzz_address_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_address_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_address_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_address_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_address_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_inv_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_inv_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DINV_DESERIALIZE=1 test_fuzz_inv_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_inv_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_inv_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_inv_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_inv_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_bloomfilter_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_bloomfilter_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOOMFILTER_DESERIALIZE=1 test_fuzz_bloomfilter_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_bloomfilter_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_bloomfilter_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_bloomfilter_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_bloomfilter_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_diskblockindex_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_diskblockindex_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DDISKBLOCKINDEX_DESERIALIZE=1 test_fuzz_diskblockindex_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_diskblockindex_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_diskblockindex_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_diskblockindex_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_diskblockindex_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_txoutcompressor_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_txoutcompressor_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DTXOUTCOMPRESSOR_DESERIALIZE=1 test_fuzz_txoutcompressor_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_txoutcompressor_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_txoutcompressor_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_txoutcompressor_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_txoutcompressor_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blocktransactions_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocktransactions_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONS_DESERIALIZE=1 test_fuzz_blocktransactions_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactions_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocktransactions_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blocktransactions_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blocktransactions_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/test_bitcoin_fuzzy.cpp +test_fuzz_blocktransactionsrequest_deserialize_SOURCES = $(FUZZ_SUITE) test/fuzz/deserialize.cpp test_fuzz_blocktransactionsrequest_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DBLOCKTRANSACTIONSREQUEST_DESERIALIZE=1 test_fuzz_blocktransactionsrequest_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_blocktransactionsrequest_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -test_fuzz_blocktransactionsrequest_deserialize_LDADD = \ - $(LIBUNIVALUE) \ - $(LIBBITCOIN_SERVER) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ - $(LIBBITCOIN_CRYPTO_SHANI) \ - $(LIBSECP256K1) -test_fuzz_blocktransactionsrequest_deserialize_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) +test_fuzz_blocktransactionsrequest_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) endif # ENABLE_FUZZ nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) diff --git a/src/addrman.cpp b/src/addrman.cpp index 06c342ba73..8a5f78d1c5 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -239,7 +239,9 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime // Will moving this address into tried evict another entry? if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { - LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table, moving %s to m_tried_collisions=%d\n", addr.ToString(), m_tried_collisions.size()); + // Output the entry we'd be colliding with, for debugging purposes + auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); + LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table (%s), moving %s to m_tried_collisions=%d\n", colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", addr.ToString(), m_tried_collisions.size()); if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { m_tried_collisions.insert(nId); } @@ -561,12 +563,19 @@ void CAddrMan::ResolveCollisions_() // Give address at least 60 seconds to successfully connect if (GetAdjustedTime() - info_old.nLastTry > 60) { - LogPrint(BCLog::ADDRMAN, "Swapping %s for %s in tried table\n", info_new.ToString(), info_old.ToString()); + LogPrint(BCLog::ADDRMAN, "Replacing %s with %s in tried table\n", info_old.ToString(), info_new.ToString()); // Replaces an existing address already in the tried table with the new address Good_(info_new, false, GetAdjustedTime()); erase_collision = true; } + } else if (GetAdjustedTime() - info_new.nLastSuccess > ADDRMAN_TEST_WINDOW) { + // If the collision hasn't resolved in some reasonable amount of time, + // just evict the old entry -- we must not be able to + // connect to it for some reason. + LogPrint(BCLog::ADDRMAN, "Unable to test; replacing %s with %s in tried table anyway\n", info_old.ToString(), info_new.ToString()); + Good_(info_new, false, GetAdjustedTime()); + erase_collision = true; } } else { // Collision is not actually a collision anymore Good_(info_new, false, GetAdjustedTime()); diff --git a/src/addrman.h b/src/addrman.h index 003bd059f8..e54184ce35 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -166,6 +166,9 @@ public: //! the maximum number of tried addr collisions to store #define ADDRMAN_SET_TRIED_COLLISION_SIZE 10 +//! the maximum time we'll spend trying to resolve a tried table collision, in seconds +static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes + /** * Stochastical (IP) address manager */ diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index b804a84478..d67b2c5bc7 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -24,7 +24,8 @@ static const int64_t DEFAULT_PLOT_HEIGHT = 768; static void SetupBenchArgs() { - gArgs.AddArg("-?", "Print this help message and exit", false, OptionsCategory::OPTIONS); + SetupHelpOptions(gArgs); + gArgs.AddArg("-list", "List benchmarks without executing them. Can be combined with -scaling and -filter", false, OptionsCategory::OPTIONS); gArgs.AddArg("-evals=<n>", strprintf("Number of measurement evaluations to perform. (default: %u)", DEFAULT_BENCH_EVALUATIONS), false, OptionsCategory::OPTIONS); gArgs.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), false, OptionsCategory::OPTIONS); @@ -33,10 +34,6 @@ static void SetupBenchArgs() gArgs.AddArg("-plot-plotlyurl=<uri>", strprintf("URL to use for plotly.js (default: %s)", DEFAULT_PLOT_PLOTLYURL), false, OptionsCategory::OPTIONS); gArgs.AddArg("-plot-width=<x>", strprintf("Plot width in pixel (default: %u)", DEFAULT_PLOT_WIDTH), false, OptionsCategory::OPTIONS); gArgs.AddArg("-plot-height=<x>", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), false, OptionsCategory::OPTIONS); - - // Hidden - gArgs.AddArg("-h", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-help", "", false, OptionsCategory::HIDDEN); } static fs::path SetDataDir() diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 8133fd9d65..cd4543c012 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -73,10 +73,12 @@ static void AssembleBlock(benchmark::State& state) boost::thread_group thread_group; CScheduler scheduler; { + LOCK(cs_main); ::pblocktree.reset(new CBlockTreeDB(1 << 20, true)); ::pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); ::pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - + } + { const CChainParams& chainparams = Params(); thread_group.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index e0854e2c62..1f6840d813 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -37,10 +37,12 @@ static void DuplicateInputs(benchmark::State& state) CScheduler scheduler; const CChainParams& chainparams = Params(); { + LOCK(cs_main); ::pblocktree.reset(new CBlockTreeDB(1 << 20, true)); ::pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); ::pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - + } + { thread_group.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); LoadGenesisBlock(chainparams); diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp new file mode 100644 index 0000000000..67d8a25564 --- /dev/null +++ b/src/bench/rpc_mempool.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <policy/policy.h> +#include <rpc/blockchain.h> +#include <txmempool.h> + +#include <univalue.h> + +#include <list> +#include <vector> + +static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) +{ + LockPoints lp; + pool.addUnchecked(CTxMemPoolEntry(tx, fee, /* time */ 0, /* height */ 1, /* spendsCoinbase */ false, /* sigOpCost */ 4, lp)); +} + +static void RpcMempool(benchmark::State& state) +{ + CTxMemPool pool; + LOCK2(cs_main, pool.cs); + + for (int i = 0; i < 1000; ++i) { + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(1); + tx.vin[0].scriptSig = CScript() << OP_1; + tx.vin[0].scriptWitness.stack.push_back({1}); + tx.vout.resize(1); + tx.vout[0].scriptPubKey = CScript() << OP_1 << OP_EQUAL; + tx.vout[0].nValue = i; + const CTransactionRef tx_r{MakeTransactionRef(tx)}; + AddTx(tx_r, /* fee */ i, pool); + } + + while (state.KeepRunning()) { + (void)MempoolToJSON(pool, /*verbose*/ true); + } +} + +BENCHMARK(RpcMempool, 40); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 7cdf61ad35..1009a771f8 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -34,11 +34,12 @@ static const int CONTINUE_EXECUTION=-1; static void SetupCliArgs() { + SetupHelpOptions(gArgs); + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); const auto regtestBaseParams = CreateBaseChainParams(CBaseChainParams::REGTEST); - gArgs.AddArg("-?", "This help message", false, OptionsCategory::OPTIONS); gArgs.AddArg("-version", "Print version and exit", false, OptionsCategory::OPTIONS); gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), false, OptionsCategory::OPTIONS); gArgs.AddArg("-datadir=<dir>", "Specify data directory", false, OptionsCategory::OPTIONS); @@ -55,10 +56,6 @@ static void SetupCliArgs() gArgs.AddArg("-rpcwallet=<walletname>", "Send RPC for non-default wallet on RPC server (needs to exactly match corresponding -wallet option passed to bitcoind). This changes the RPC endpoint used, e.g. http://127.0.0.1:8332/wallet/<walletname>", false, OptionsCategory::OPTIONS); gArgs.AddArg("-stdin", "Read extra arguments from standard input, one per line until EOF/Ctrl-D (recommended for sensitive information such as passphrases). When combined with -stdinrpcpass, the first line from standard input is used for the RPC password.", false, OptionsCategory::OPTIONS); gArgs.AddArg("-stdinrpcpass", "Read RPC password from standard input as a single line. When combined with -stdin, the first line from standard input is used for the RPC password.", false, OptionsCategory::OPTIONS); - - // Hidden - gArgs.AddArg("-h", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-help", "", false, OptionsCategory::HIDDEN); } /** libevent event log callback */ @@ -256,16 +253,12 @@ public: } result.pushKV("version", batch[ID_NETWORKINFO]["result"]["version"]); result.pushKV("protocolversion", batch[ID_NETWORKINFO]["result"]["protocolversion"]); - if (!batch[ID_WALLETINFO].isNull()) { - result.pushKV("walletversion", batch[ID_WALLETINFO]["result"]["walletversion"]); - result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); - } result.pushKV("blocks", batch[ID_BLOCKCHAININFO]["result"]["blocks"]); result.pushKV("timeoffset", batch[ID_NETWORKINFO]["result"]["timeoffset"]); result.pushKV("connections", batch[ID_NETWORKINFO]["result"]["connections"]); result.pushKV("proxy", batch[ID_NETWORKINFO]["result"]["networks"][0]["proxy"]); result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); - result.pushKV("testnet", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"].get_str() == "test")); + result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); if (!batch[ID_WALLETINFO].isNull()) { result.pushKV("walletversion", batch[ID_WALLETINFO]["result"]["walletversion"]); result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 9d1309cde3..4be89aab6c 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -35,7 +35,8 @@ const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; static void SetupBitcoinTxArgs() { - gArgs.AddArg("-?", "This help message", false, OptionsCategory::OPTIONS); + SetupHelpOptions(gArgs); + gArgs.AddArg("-create", "Create new, empty TX.", false, OptionsCategory::OPTIONS); gArgs.AddArg("-json", "Select JSON output", false, OptionsCategory::OPTIONS); gArgs.AddArg("-txid", "Output only the hex-encoded transaction id of the resultant transaction.", false, OptionsCategory::OPTIONS); @@ -66,10 +67,6 @@ static void SetupBitcoinTxArgs() gArgs.AddArg("load=NAME:FILENAME", "Load JSON file FILENAME into register NAME", false, OptionsCategory::REGISTER_COMMANDS); gArgs.AddArg("set=NAME:JSON-STRING", "Set register NAME to given JSON-STRING", false, OptionsCategory::REGISTER_COMMANDS); - - // Hidden - gArgs.AddArg("-h", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-help", "", false, OptionsCategory::HIDDEN); } // diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 11caa0dc6c..32a539aac6 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -20,9 +20,9 @@ const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; static void SetupWalletToolArgs() { + SetupHelpOptions(gArgs); SetupChainParamsBaseOptions(); - gArgs.AddArg("-?", "This help message", false, OptionsCategory::OPTIONS); gArgs.AddArg("-datadir=<dir>", "Specify data directory", false, OptionsCategory::OPTIONS); gArgs.AddArg("-wallet=<wallet-name>", "Specify wallet name", false, OptionsCategory::OPTIONS); gArgs.AddArg("-debug=<category>", "Output debugging information (default: 0).", false, OptionsCategory::DEBUG_TEST); @@ -30,10 +30,6 @@ static void SetupWalletToolArgs() gArgs.AddArg("info", "Get wallet info", false, OptionsCategory::COMMANDS); gArgs.AddArg("create", "Create new wallet file", false, OptionsCategory::COMMANDS); - - // Hidden - gArgs.AddArg("-h", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-help", "", false, OptionsCategory::HIDDEN); } static bool WalletAppInit(int argc, char* argv[]) diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index bcf24047ff..e15213c552 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -251,6 +251,8 @@ bool BlockFilter::BuildParams(GCSFilter::Params& params) const params.m_P = BASIC_FILTER_P; params.m_M = BASIC_FILTER_M; return true; + case BlockFilterType::INVALID: + return false; } return false; diff --git a/src/blockfilter.h b/src/blockfilter.h index 4d1f51dd60..e5e087ed5a 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -83,9 +83,10 @@ public: constexpr uint8_t BASIC_FILTER_P = 19; constexpr uint32_t BASIC_FILTER_M = 784931; -enum BlockFilterType : uint8_t +enum class BlockFilterType : uint8_t { BASIC = 0, + INVALID = 255, }; /** @@ -95,7 +96,7 @@ enum BlockFilterType : uint8_t class BlockFilter { private: - BlockFilterType m_filter_type; + BlockFilterType m_filter_type = BlockFilterType::INVALID; uint256 m_block_hash; GCSFilter m_filter; diff --git a/src/chain.h b/src/chain.h index 5a6f10b84f..2b6d2d082c 100644 --- a/src/chain.h +++ b/src/chain.h @@ -8,6 +8,7 @@ #include <arith_uint256.h> #include <consensus/params.h> +#include <flatfile.h> #include <primitives/block.h> #include <tinyformat.h> #include <uint256.h> @@ -90,46 +91,6 @@ public: } }; -struct CDiskBlockPos -{ - int nFile; - unsigned int nPos; - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nPos)); - } - - CDiskBlockPos() { - SetNull(); - } - - CDiskBlockPos(int nFileIn, unsigned int nPosIn) { - nFile = nFileIn; - nPos = nPosIn; - } - - friend bool operator==(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return (a.nFile == b.nFile && a.nPos == b.nPos); - } - - friend bool operator!=(const CDiskBlockPos &a, const CDiskBlockPos &b) { - return !(a == b); - } - - void SetNull() { nFile = -1; nPos = 0; } - bool IsNull() const { return (nFile == -1); } - - std::string ToString() const - { - return strprintf("CDiskBlockPos(nFile=%i, nPos=%i)", nFile, nPos); - } - -}; - enum BlockStatus: uint32_t { //! Unused. BLOCK_VALID_UNKNOWN = 0, @@ -266,8 +227,8 @@ public: nNonce = block.nNonce; } - CDiskBlockPos GetBlockPos() const { - CDiskBlockPos ret; + FlatFilePos GetBlockPos() const { + FlatFilePos ret; if (nStatus & BLOCK_HAVE_DATA) { ret.nFile = nFile; ret.nPos = nDataPos; @@ -275,8 +236,8 @@ public: return ret; } - CDiskBlockPos GetUndoPos() const { - CDiskBlockPos ret; + FlatFilePos GetUndoPos() const { + FlatFilePos ret; if (nStatus & BLOCK_HAVE_UNDO) { ret.nFile = nFile; ret.nPos = nUndoPos; diff --git a/src/chainparams.cpp b/src/chainparams.cpp index d3972fdb89..b8e0ea23dd 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -91,10 +91,10 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1510704000; // November 15th, 2017. // The best chain should have at least this much work. - consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000028822fef1c230963535a90d"); + consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000051dc8b82f450202ecb3d471"); // By default assume that the signatures in ancestors of this block are valid. - consensus.defaultAssumeValid = uint256S("0x0000000000000000002e63058c023a9a1de233554f28c7b21380b6c9003f36a8"); //534292 + consensus.defaultAssumeValid = uint256S("0x0000000000000000000f1c54590ee18d15ec70e68c8cd4cfbadb1b4f11697eee"); //563378 /** * The message start string is designed to be unlikely to occur in normal data. @@ -107,7 +107,7 @@ public: pchMessageStart[3] = 0xd9; nDefaultPort = 8333; nPruneAfterHeight = 100000; - m_assumed_blockchain_size = 200; + m_assumed_blockchain_size = 240; m_assumed_chain_state_size = 3; genesis = CreateGenesisBlock(1231006505, 2083236893, 0x1d00ffff, 1, 50 * COIN); @@ -162,10 +162,10 @@ public: }; chainTxData = ChainTxData{ - // Data from rpc: getchaintxstats 4096 0000000000000000002e63058c023a9a1de233554f28c7b21380b6c9003f36a8 - /* nTime */ 1532884444, - /* nTxCount */ 331282217, - /* dTxRate */ 2.4 + // Data from rpc: getchaintxstats 4096 0000000000000000000f1c54590ee18d15ec70e68c8cd4cfbadb1b4f11697eee + /* nTime */ 1550374134, + /* nTxCount */ 383732546, + /* dTxRate */ 3.685496590998308 }; /* disable fallback fee on mainnet */ @@ -219,7 +219,7 @@ public: pchMessageStart[3] = 0x07; nDefaultPort = 18333; nPruneAfterHeight = 1000; - m_assumed_blockchain_size = 20; + m_assumed_blockchain_size = 30; m_assumed_chain_state_size = 2; genesis = CreateGenesisBlock(1296688602, 414098458, 0x1d00ffff, 1, 50 * COIN); diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h new file mode 100644 index 0000000000..820c9b93d9 --- /dev/null +++ b/src/compat/assumptions.h @@ -0,0 +1,49 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Compile-time verification of assumptions we make. + +#ifndef BITCOIN_COMPAT_ASSUMPTIONS_H +#define BITCOIN_COMPAT_ASSUMPTIONS_H + +#include <limits> + +// Assumption: We assume that the macro NDEBUG is not defined. +// Example(s): We use assert(...) extensively with the assumption of it never +// being a noop at runtime. +#if defined(NDEBUG) +# error "Bitcoin cannot be compiled without assertions." +#endif + +// Assumption: We assume the floating-point types to fulfill the requirements of +// IEC 559 (IEEE 754) standard. +// Example(s): Floating-point division by zero in ConnectBlock, CreateTransaction +// and EstimateMedianVal. +static_assert(std::numeric_limits<float>::is_iec559, "IEEE 754 float assumed"); +static_assert(std::numeric_limits<double>::is_iec559, "IEEE 754 double assumed"); + +// Assumption: We assume eight bits per byte (obviously, but remember: don't +// trust -- verify!). +// Example(s): Everywhere :-) +static_assert(std::numeric_limits<unsigned char>::digits == 8, "8-bit byte assumed"); + +// Assumption: We assume floating-point widths. +// Example(s): Type punning in serialization code (ser_{float,double}_to_uint{32,64}). +static_assert(sizeof(float) == 4, "32-bit float assumed"); +static_assert(sizeof(double) == 8, "64-bit double assumed"); + +// Assumption: We assume integer widths. +// Example(s): GetSizeOfCompactSize and WriteCompactSize in the serialization +// code. +static_assert(sizeof(short) == 2, "16-bit short assumed"); +static_assert(sizeof(int) == 4, "32-bit int assumed"); + +// Some important things we are NOT assuming (non-exhaustive list): +// * We are NOT assuming a specific value for sizeof(std::size_t). +// * We are NOT assuming a specific value for std::endian::native. +// * We are NOT assuming a specific value for std::locale("").name(). +// * We are NOT assuming a specific value for std::numeric_limits<char>::is_signed. + +#endif // BITCOIN_COMPAT_ASSUMPTIONS_H diff --git a/src/core_io.h b/src/core_io.h index 6f87161f46..ae377eb6e8 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -37,7 +37,11 @@ bool DecodeHexBlockHeader(CBlockHeader&, const std::string& hex_header); */ bool ParseHashStr(const std::string& strHex, uint256& result); std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strName); -NODISCARD bool DecodePSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error); + +//! Decode a base64ed PSBT into a PartiallySignedTransaction +NODISCARD bool DecodeBase64PSBT(PartiallySignedTransaction& decoded_psbt, const std::string& base64_psbt, std::string& error); +//! Decode a raw (binary blob) PSBT into a PartiallySignedTransaction +NODISCARD bool DecodeRawPSBT(PartiallySignedTransaction& decoded_psbt, const std::string& raw_psbt, std::string& error); int ParseSighashString(const UniValue& sighash); // core_write.cpp diff --git a/src/core_read.cpp b/src/core_read.cpp index 3b82b2853c..536a7f4f17 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -4,6 +4,7 @@ #include <core_io.h> +#include <psbt.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <script/script.h> @@ -176,10 +177,20 @@ bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) return true; } -bool DecodePSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error) +bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base64_tx, std::string& error) { - std::vector<unsigned char> tx_data = DecodeBase64(base64_tx.c_str()); - CDataStream ss_data(tx_data, SER_NETWORK, PROTOCOL_VERSION); + bool invalid; + std::string tx_data = DecodeBase64(base64_tx, &invalid); + if (invalid) { + error = "invalid base64"; + return false; + } + return DecodeRawPSBT(psbt, tx_data, error); +} + +bool DecodeRawPSBT(PartiallySignedTransaction& psbt, const std::string& tx_data, std::string& error) +{ + CDataStream ss_data(tx_data.data(), tx_data.data() + tx_data.size(), SER_NETWORK, PROTOCOL_VERSION); try { ss_data >> psbt; if (!ss_data.empty()) { diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 9211a7596b..8a76021a5b 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -8,6 +8,10 @@ class CWallet; +namespace interfaces { +class Chain; +} + class DummyWalletInit : public WalletInitInterface { public: @@ -43,6 +47,11 @@ std::vector<std::shared_ptr<CWallet>> GetWallets() throw std::logic_error("Wallet function called in non-wallet build."); } +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::string& warning) +{ + throw std::logic_error("Wallet function called in non-wallet build."); +} + namespace interfaces { class Wallet; diff --git a/src/flatfile.cpp b/src/flatfile.cpp new file mode 100644 index 0000000000..8a8f7b681c --- /dev/null +++ b/src/flatfile.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <stdexcept> + +#include <flatfile.h> +#include <logging.h> +#include <tinyformat.h> +#include <util/system.h> + +FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) : + m_dir(std::move(dir)), + m_prefix(prefix), + m_chunk_size(chunk_size) +{ + if (chunk_size == 0) { + throw std::invalid_argument("chunk_size must be positive"); + } +} + +std::string FlatFilePos::ToString() const +{ + return strprintf("FlatFilePos(nFile=%i, nPos=%i)", nFile, nPos); +} + +fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const +{ + return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); +} + +FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) +{ + if (pos.IsNull()) { + return nullptr; + } + fs::path path = FileName(pos); + fs::create_directories(path.parent_path()); + FILE* file = fsbridge::fopen(path, read_only ? "rb": "rb+"); + if (!file && !read_only) + file = fsbridge::fopen(path, "wb+"); + if (!file) { + LogPrintf("Unable to open file %s\n", path.string()); + return nullptr; + } + if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) { + LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); + fclose(file); + return nullptr; + } + return file; +} + +size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space) +{ + out_of_space = false; + + unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size; + unsigned int n_new_chunks = (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size; + if (n_new_chunks > n_old_chunks) { + size_t old_size = pos.nPos; + size_t new_size = n_new_chunks * m_chunk_size; + size_t inc_size = new_size - old_size; + + if (CheckDiskSpace(m_dir, inc_size)) { + FILE *file = Open(pos); + if (file) { + LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile); + AllocateFileRange(file, pos.nPos, inc_size); + fclose(file); + return inc_size; + } + } else { + out_of_space = true; + } + } + return 0; +} + +bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize) +{ + FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos + if (!file) { + return error("%s: failed to open file %d", __func__, pos.nFile); + } + if (finalize && !TruncateFile(file, pos.nPos)) { + fclose(file); + return error("%s: failed to truncate file %d", __func__, pos.nFile); + } + if (!FileCommit(file)) { + fclose(file); + return error("%s: failed to commit file %d", __func__, pos.nFile); + } + + fclose(file); + return true; +} diff --git a/src/flatfile.h b/src/flatfile.h new file mode 100644 index 0000000000..374ceff411 --- /dev/null +++ b/src/flatfile.h @@ -0,0 +1,96 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_FLATFILE_H +#define BITCOIN_FLATFILE_H + +#include <string> + +#include <fs.h> +#include <serialize.h> + +struct FlatFilePos +{ + int nFile; + unsigned int nPos; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT(nPos)); + } + + FlatFilePos() : nFile(-1), nPos(0) {} + + FlatFilePos(int nFileIn, unsigned int nPosIn) : + nFile(nFileIn), + nPos(nPosIn) + {} + + friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) { + return (a.nFile == b.nFile && a.nPos == b.nPos); + } + + friend bool operator!=(const FlatFilePos &a, const FlatFilePos &b) { + return !(a == b); + } + + void SetNull() { nFile = -1; nPos = 0; } + bool IsNull() const { return (nFile == -1); } + + std::string ToString() const; +}; + +/** + * FlatFileSeq represents a sequence of numbered files storing raw data. This class facilitates + * access to and efficient management of these files. + */ +class FlatFileSeq +{ +private: + const fs::path m_dir; + const char* const m_prefix; + const size_t m_chunk_size; + +public: + /** + * Constructor + * + * @param dir The base directory that all files live in. + * @param prefix A short prefix given to all file names. + * @param chunk_size Disk space is pre-allocated in multiples of this amount. + */ + FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size); + + /** Get the name of the file at the given position. */ + fs::path FileName(const FlatFilePos& pos) const; + + /** Open a handle to the file at the given position. */ + FILE* Open(const FlatFilePos& pos, bool read_only = false); + + /** + * Allocate additional space in a file after the given starting position. The amount allocated + * will be the minimum multiple of the sequence chunk size greater than add_size. + * + * @param[in] pos The starting position that bytes will be allocated after. + * @param[in] add_size The minimum number of bytes to be allocated. + * @param[out] out_of_space Whether the allocation failed due to insufficient disk space. + * @return The number of bytes successfully allocated. + */ + size_t Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space); + + /** + * Commit a file to disk, and optionally truncate off extra pre-allocated bytes if final. + * + * @param[in] pos The first unwritten position in the file to be flushed. + * @param[in] finalize True if no more data will be written to this file. + * @return true on success, false on failure. + */ + bool Flush(const FlatFilePos& pos, bool finalize = false); +}; + +#endif // BITCOIN_FLATFILE_H diff --git a/src/fs.cpp b/src/fs.cpp index 3c8f4c0247..f937f64e04 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -160,6 +160,7 @@ static std::string openmodeToStr(std::ios_base::openmode mode) void ifstream::open(const fs::path& p, std::ios_base::openmode mode) { close(); + mode |= std::ios_base::in; m_file = fsbridge::fopen(p, openmodeToStr(mode).c_str()); if (m_file == nullptr) { return; @@ -183,6 +184,7 @@ void ifstream::close() void ofstream::open(const fs::path& p, std::ios_base::openmode mode) { close(); + mode |= std::ios_base::out; m_file = fsbridge::fopen(p, openmodeToStr(mode).c_str()); if (m_file == nullptr) { return; diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 10bc8419dd..7367ec7cb6 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -16,7 +16,7 @@ constexpr char DB_TXINDEX_BLOCK = 'T'; std::unique_ptr<TxIndex> g_txindex; -struct CDiskTxPos : public CDiskBlockPos +struct CDiskTxPos : public FlatFilePos { unsigned int nTxOffset; // after header @@ -24,11 +24,11 @@ struct CDiskTxPos : public CDiskBlockPos template <typename Stream, typename Operation> inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CDiskBlockPos, *this); + READWRITEAS(FlatFilePos, *this); READWRITE(VARINT(nTxOffset)); } - CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { } CDiskTxPos() { @@ -36,7 +36,7 @@ struct CDiskTxPos : public CDiskBlockPos } void SetNull() { - CDiskBlockPos::SetNull(); + FlatFilePos::SetNull(); nTxOffset = 0; } }; diff --git a/src/init.cpp b/src/init.cpp index 8fcf94b49b..0898d0ff25 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -53,6 +53,8 @@ #include <stdio.h> #ifndef WIN32 +#include <attributes.h> +#include <cerrno> #include <signal.h> #include <sys/stat.h> #endif @@ -92,6 +94,32 @@ std::unique_ptr<BanMan> g_banman; static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; +/** + * The PID file facilities. + */ +static const char* BITCOIN_PID_FILENAME = "bitcoind.pid"; + +static fs::path GetPidFile() +{ + return AbsPathForConfigVal(fs::path(gArgs.GetArg("-pid", BITCOIN_PID_FILENAME))); +} + +NODISCARD static bool CreatePidFile() +{ + FILE* file = fsbridge::fopen(GetPidFile(), "w"); + if (file) { +#ifdef WIN32 + fprintf(file, "%d\n", GetCurrentProcessId()); +#else + fprintf(file, "%d\n", getpid()); +#endif + fclose(file); + return true; + } else { + return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile().string(), std::strerror(errno))); + } +} + ////////////////////////////////////////////////////////////////////////////// // // Shutdown @@ -260,13 +288,13 @@ void Shutdown(InitInterfaces& interfaces) } #endif -#ifndef WIN32 try { - fs::remove(GetPidFile()); + if (!fs::remove(GetPidFile())) { + LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); + } } catch (const fs::filesystem_error& e) { - LogPrintf("%s: Unable to remove pidfile: %s\n", __func__, e.what()); + LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); } -#endif interfaces.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); @@ -326,6 +354,9 @@ static void OnRPCStopped() void SetupServerArgs() { + SetupHelpOptions(gArgs); + gArgs.AddArg("-help-debug", "Print help message with debugging options and exit", false, OptionsCategory::DEBUG_TEST); // server-only for now + const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); const auto regtestBaseParams = CreateBaseChainParams(CBaseChainParams::REGTEST); @@ -334,14 +365,11 @@ void SetupServerArgs() const auto regtestChainParams = CreateChainParams(CBaseChainParams::REGTEST); // Hidden Options - std::vector<std::string> hidden_args = {"-h", "-help", + std::vector<std::string> hidden_args = { "-dbcrashratio", "-forcecompactdb", // GUI args. These will be overwritten by SetupUIArgs for the GUI "-allowselfsignedrootcertificates", "-choosedatadir", "-lang=<lang>", "-min", "-resetguisettings", "-rootcertificates=<file>", "-splash", "-uiplatform"}; - // Set all of the args and their help - // When adding new options to the categories, please keep and ensure alphabetical ordering. - gArgs.AddArg("-?", "Print this help message and exit", false, OptionsCategory::OPTIONS); gArgs.AddArg("-version", "Print version and exit", false, OptionsCategory::OPTIONS); gArgs.AddArg("-alertnotify=<cmd>", "Execute command when a relevant alert is received or we see a really long fork (%s in cmd is replaced by message)", false, OptionsCategory::OPTIONS); gArgs.AddArg("-assumevalid=<hex>", strprintf("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)", defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex()), false, OptionsCategory::OPTIONS); @@ -352,7 +380,7 @@ void SetupServerArgs() gArgs.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), false, OptionsCategory::OPTIONS); gArgs.AddArg("-datadir=<dir>", "Specify data directory", false, OptionsCategory::OPTIONS); gArgs.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), true, OptionsCategory::OPTIONS); - gArgs.AddArg("-dbcache=<n>", strprintf("Set database cache size in MiB (%d to %d, default: %d)", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS); + gArgs.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), false, OptionsCategory::OPTIONS); gArgs.AddArg("-debuglogfile=<file>", strprintf("Specify location of debug log file. Relative paths will be prefixed by a net-specific datadir location. (-nodebuglogfile to disable; default: %s)", DEFAULT_DEBUGLOGFILE), false, OptionsCategory::OPTIONS); gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), true, OptionsCategory::OPTIONS); gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", false, OptionsCategory::OPTIONS); @@ -364,11 +392,7 @@ void SetupServerArgs() gArgs.AddArg("-par=<n>", strprintf("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)", -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), false, OptionsCategory::OPTIONS); gArgs.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), false, OptionsCategory::OPTIONS); -#ifndef WIN32 gArgs.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), false, OptionsCategory::OPTIONS); -#else - hidden_args.emplace_back("-pid"); -#endif gArgs.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), false, OptionsCategory::OPTIONS); @@ -470,7 +494,6 @@ void SetupServerArgs() gArgs.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + ListLogCategories() + ".", false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."), false, OptionsCategory::DEBUG_TEST); - gArgs.AddArg("-help-debug", "Print help message with debugging options and exit", false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), true, OptionsCategory::DEBUG_TEST); @@ -645,8 +668,8 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) if (fReindex) { int nFile = 0; while (true) { - CDiskBlockPos pos(nFile, 0); - if (!fs::exists(GetBlockPosFilename(pos, "blk"))) + FlatFilePos pos(nFile, 0); + if (!fs::exists(GetBlockPosFilename(pos))) break; // No block files left to reindex FILE *file = OpenBlockFile(pos, true); if (!file) @@ -816,7 +839,7 @@ void InitParameterInteraction() // Warn if unrecognized section name are present in the config file. for (const auto& section : gArgs.GetUnrecognizedSections()) { - InitWarning(strprintf(_("Section [%s] is not recognized."), section)); + InitWarning(strprintf("%s:%i " + _("Section [%s] is not recognized."), section.m_file, section.m_line, section.m_name)); } } @@ -1098,6 +1121,22 @@ bool AppInitParameterInteraction() dustRelayFee = CFeeRate(n); } + // This is required by both the wallet and node + if (gArgs.IsArgSet("-maxtxfee")) + { + CAmount nMaxFee = 0; + if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) + return InitError(AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", ""))); + if (nMaxFee > HIGH_MAX_TX_FEE) + InitWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); + maxTxFee = nMaxFee; + if (CFeeRate(maxTxFee, 1000) < ::minRelayTxFee) + { + return InitError(strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), + gArgs.GetArg("-maxtxfee", ""), ::minRelayTxFee.ToString())); + } + } + fRequireStandard = !gArgs.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (chainparams.RequireStandard() && !fRequireStandard) return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); @@ -1185,9 +1224,10 @@ bool AppInitMain(InitInterfaces& interfaces) { const CChainParams& chainparams = Params(); // ********************************************************* Step 4a: application initialization -#ifndef WIN32 - CreatePidFile(GetPidFile(), getpid()); -#endif + if (!CreatePidFile()) { + // Detailed error printed inside CreatePidFile(). + return false; + } if (LogInstance().m_print_to_file) { if (gArgs.GetBoolArg("-shrinkdebugfile", LogInstance().DefaultShrinkDebugFile())) { // Do this first since it both loads a bunch of debug.log into memory, @@ -1428,11 +1468,11 @@ bool AppInitMain(InitInterfaces& interfaces) uiInterface.InitMessage(_("Loading block index...")); - LOCK(cs_main); - do { const int64_t load_block_index_start_time = GetTimeMillis(); + bool is_coinsview_empty; try { + LOCK(cs_main); UnloadBlockIndex(); pcoinsTip.reset(); pcoinsdbview.reset(); @@ -1504,7 +1544,7 @@ bool AppInitMain(InitInterfaces& interfaces) // The on-disk coinsdb is now in a good state, create the cache pcoinsTip.reset(new CCoinsViewCache(pcoinscatcher.get())); - bool is_coinsview_empty = fReset || fReindexChainState || pcoinsTip->GetBestBlock().IsNull(); + is_coinsview_empty = fReset || fReindexChainState || pcoinsTip->GetBestBlock().IsNull(); if (!is_coinsview_empty) { // LoadChainTip sets chainActive based on pcoinsTip's best block if (!LoadChainTip(chainparams)) { @@ -1513,18 +1553,25 @@ bool AppInitMain(InitInterfaces& interfaces) } assert(chainActive.Tip() != nullptr); } + } catch (const std::exception& e) { + LogPrintf("%s\n", e.what()); + strLoadError = _("Error opening block database"); + break; + } - if (!fReset) { - // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. - // It both disconnects blocks based on chainActive, and drops block data in - // mapBlockIndex based on lack of available witness data. - uiInterface.InitMessage(_("Rewinding blocks...")); - if (!RewindBlockIndex(chainparams)) { - strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain"); - break; - } + if (!fReset) { + // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. + // It both disconnects blocks based on chainActive, and drops block data in + // mapBlockIndex based on lack of available witness data. + uiInterface.InitMessage(_("Rewinding blocks...")); + if (!RewindBlockIndex(chainparams)) { + strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain"); + break; } + } + try { + LOCK(cs_main); if (!is_coinsview_empty) { uiInterface.InitMessage(_("Verifying blocks...")); if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { @@ -1629,11 +1676,11 @@ bool AppInitMain(InitInterfaces& interfaces) // ********************************************************* Step 11: import blocks - if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ false)) { + if (!CheckDiskSpace(GetDataDir())) { InitError(strprintf(_("Error: Disk space is low for %s"), GetDataDir())); return false; } - if (!CheckDiskSpace(/* additional_bytes */ 0, /* blocks_dir */ true)) { + if (!CheckDiskSpace(GetBlocksDir())) { InitError(strprintf(_("Error: Disk space is low for %s"), GetBlocksDir())); return false; } diff --git a/src/interfaces/README.md b/src/interfaces/README.md index 57d41df746..f77d172153 100644 --- a/src/interfaces/README.md +++ b/src/interfaces/README.md @@ -2,9 +2,9 @@ The following interfaces are defined here: -* [`Chain`](chain.h) — used by wallet to access blockchain and mempool state. Added in [#10973](https://github.com/bitcoin/bitcoin/pull/10973). +* [`Chain`](chain.h) — used by wallet to access blockchain and mempool state. Added in [#14437](https://github.com/bitcoin/bitcoin/pull/14437), [#14711](https://github.com/bitcoin/bitcoin/pull/14711), [#15288](https://github.com/bitcoin/bitcoin/pull/15288), and [#10973](https://github.com/bitcoin/bitcoin/pull/10973). -* [`ChainClient`](chain.h) — used by node to start & stop `Chain` clients. Added in [#10973](https://github.com/bitcoin/bitcoin/pull/10973). +* [`ChainClient`](chain.h) — used by node to start & stop `Chain` clients. Added in [#14437](https://github.com/bitcoin/bitcoin/pull/14437). * [`Node`](node.h) — used by GUI to start & stop bitcoin node. Added in [#10244](https://github.com/bitcoin/bitcoin/pull/10244). diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index da810bc5e6..2eecea28d0 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -6,8 +6,19 @@ #include <chain.h> #include <chainparams.h> +#include <interfaces/wallet.h> +#include <net.h> +#include <policy/fees.h> +#include <policy/policy.h> +#include <policy/rbf.h> #include <primitives/block.h> +#include <primitives/transaction.h> +#include <protocol.h> #include <sync.h> +#include <threadsafety.h> +#include <timedata.h> +#include <txmempool.h> +#include <ui_interface.h> #include <uint256.h> #include <util/system.h> #include <validation.h> @@ -132,6 +143,17 @@ class LockImpl : public Chain::Lock } return nullopt; } + bool checkFinalTx(const CTransaction& tx) override + { + LockAnnotation lock(::cs_main); + return CheckFinalTx(tx); + } + bool submitToMemoryPool(const CTransactionRef& tx, CAmount absurd_fee, CValidationState& state) override + { + LockAnnotation lock(::cs_main); + return AcceptToMemoryPool(::mempool, state, tx, nullptr /* missing inputs */, nullptr /* txn replaced */, + false /* bypass limits */, absurd_fee); + } }; class LockingStateImpl : public LockImpl, public UniqueLock<CCriticalSection> @@ -177,6 +199,61 @@ public: LOCK(cs_main); return GuessVerificationProgress(Params().TxData(), LookupBlockIndex(block_hash)); } + RBFTransactionState isRBFOptIn(const CTransaction& tx) override + { + LOCK(::mempool.cs); + return IsRBFOptIn(tx, ::mempool); + } + bool hasDescendantsInMempool(const uint256& txid) override + { + LOCK(::mempool.cs); + auto it = ::mempool.GetIter(txid); + return it && (*it)->GetCountWithDescendants() > 1; + } + void relayTransaction(const uint256& txid) override + { + CInv inv(MSG_TX, txid); + g_connman->ForEachNode([&inv](CNode* node) { node->PushInventory(inv); }); + } + void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) override + { + ::mempool.GetTransactionAncestry(txid, ancestors, descendants); + } + bool checkChainLimits(const CTransactionRef& tx) override + { + LockPoints lp; + CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); + CTxMemPool::setEntries ancestors; + auto limit_ancestor_count = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); + auto limit_ancestor_size = gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT) * 1000; + auto limit_descendant_count = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); + auto limit_descendant_size = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; + std::string unused_error_string; + LOCK(::mempool.cs); + return ::mempool.CalculateMemPoolAncestors(entry, ancestors, limit_ancestor_count, limit_ancestor_size, + limit_descendant_count, limit_descendant_size, unused_error_string); + } + CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override + { + return ::feeEstimator.estimateSmartFee(num_blocks, calc, conservative); + } + unsigned int estimateMaxBlocks() override + { + return ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + } + CFeeRate mempoolMinFee() override + { + return ::mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + } + CAmount maxTxFee() override { return ::maxTxFee; } + bool getPruneMode() override { return ::fPruneMode; } + bool p2pEnabled() override { return g_connman != nullptr; } + bool isInitialBlockDownload() override { return IsInitialBlockDownload(); } + int64_t getAdjustedTime() override { return GetAdjustedTime(); } + void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } + void initWarning(const std::string& message) override { InitWarning(message); } + void initError(const std::string& message) override { InitError(message); } + void loadWallet(std::unique_ptr<Wallet> wallet) override { ::uiInterface.LoadWallet(wallet); } }; } // namespace diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 3a54b9164e..037e8e9ff5 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -5,21 +5,49 @@ #ifndef BITCOIN_INTERFACES_CHAIN_H #define BITCOIN_INTERFACES_CHAIN_H -#include <optional.h> +#include <optional.h> // For Optional and nullopt +#include <primitives/transaction.h> // For CTransactionRef #include <memory> +#include <stddef.h> #include <stdint.h> #include <string> #include <vector> class CBlock; +class CFeeRate; class CScheduler; +class CValidationState; class uint256; +enum class RBFTransactionState; struct CBlockLocator; +struct FeeCalculation; namespace interfaces { -//! Interface for giving wallet processes access to blockchain state. +class Wallet; + +//! Interface giving clients (wallet processes, maybe other analysis tools in +//! the future) ability to access to the chain state, receive notifications, +//! estimate fees, and submit transactions. +//! +//! TODO: Current chain methods are too low level, exposing too much of the +//! internal workings of the bitcoin node, and not being very convenient to use. +//! Chain methods should be cleaned up and simplified over time. Examples: +//! +//! * The Chain::lock() method, which lets clients delay chain tip updates +//! should be removed when clients are able to respond to updates +//! asynchronously +//! (https://github.com/bitcoin/bitcoin/pull/10973#issuecomment-380101269). +//! +//! * The relayTransactions() and submitToMemoryPool() methods could be replaced +//! with a higher-level broadcastTransaction method +//! (https://github.com/bitcoin/bitcoin/pull/14978#issuecomment-459373984). +//! +//! * The initMessages() and loadWallet() methods which the wallet uses to send +//! notifications to the GUI should go away when GUI and wallet can directly +//! communicate with each other without going through the node +//! (https://github.com/bitcoin/bitcoin/pull/15288#discussion_r253321096). class Chain { public: @@ -102,6 +130,14 @@ public: //! is guaranteed to be an ancestor of the block used to create the //! locator. virtual Optional<int> findLocatorFork(const CBlockLocator& locator) = 0; + + //! Check if transaction will be final given chain height current time. + virtual bool checkFinalTx(const CTransaction& tx) = 0; + + //! Add transaction to memory pool if the transaction fee is below the + //! amount specified by absurd_fee. Returns false if the transaction + //! could not be added due to the fee or for another reason. + virtual bool submitToMemoryPool(const CTransactionRef& tx, CAmount absurd_fee, CValidationState& state) = 0; }; //! Return Lock interface. Chain is locked when this is called, and @@ -127,6 +163,60 @@ public: //! Estimate fraction of total transactions verified if blocks up to //! the specified block hash are verified. virtual double guessVerificationProgress(const uint256& block_hash) = 0; + + //! Check if transaction is RBF opt in. + virtual RBFTransactionState isRBFOptIn(const CTransaction& tx) = 0; + + //! Check if transaction has descendants in mempool. + virtual bool hasDescendantsInMempool(const uint256& txid) = 0; + + //! Relay transaction. + virtual void relayTransaction(const uint256& txid) = 0; + + //! Calculate mempool ancestor and descendant counts for the given transaction. + virtual void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) = 0; + + //! Check if transaction will pass the mempool's chain limits. + virtual bool checkChainLimits(const CTransactionRef& tx) = 0; + + //! Estimate smart fee. + virtual CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc = nullptr) = 0; + + //! Fee estimator max target. + virtual unsigned int estimateMaxBlocks() = 0; + + //! Mempool minimum fee. + virtual CFeeRate mempoolMinFee() = 0; + + //! Node max tx fee setting (-maxtxfee). + //! This could be replaced by a per-wallet max fee, as proposed at + //! https://github.com/bitcoin/bitcoin/issues/15355 + //! But for the time being, wallets call this to access the node setting. + virtual CAmount maxTxFee() = 0; + + //! Check if pruning is enabled. + virtual bool getPruneMode() = 0; + + //! Check if p2p enabled. + virtual bool p2pEnabled() = 0; + + // Check if in IBD. + virtual bool isInitialBlockDownload() = 0; + + //! Get adjusted time. + virtual int64_t getAdjustedTime() = 0; + + //! Send init message. + virtual void initMessage(const std::string& message) = 0; + + //! Send init warning. + virtual void initWarning(const std::string& message) = 0; + + //! Send init error. + virtual void initError(const std::string& message) = 0; + + //! Send wallet load notification to the GUI. + virtual void loadWallet(std::unique_ptr<Wallet> wallet) = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index a2b89c3f90..6f7dce0c24 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -42,6 +42,7 @@ class CWallet; fs::path GetWalletDir(); std::vector<fs::path> ListWalletDir(); std::vector<std::shared_ptr<CWallet>> GetWallets(); +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::string& warning); namespace interfaces { @@ -252,6 +253,10 @@ public: } return wallets; } + std::unique_ptr<Wallet> loadWallet(const std::string& name, std::string& error, std::string& warning) override + { + return MakeWallet(LoadWallet(*m_interfaces.chain, name, error, warning)); + } std::unique_ptr<Handler> handleInitMessage(InitMessageFn fn) override { return MakeHandler(::uiInterface.InitMessage_connect(fn)); @@ -270,7 +275,7 @@ public: } std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) override { - return MakeHandler(::uiInterface.LoadWallet_connect([fn](std::shared_ptr<CWallet> wallet) { fn(MakeWallet(wallet)); })); + return MakeHandler(::uiInterface.LoadWallet_connect([fn](std::unique_ptr<Wallet>& wallet) { fn(std::move(wallet)); })); } std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) override { diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 54c2d78338..76b93af234 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -192,6 +192,11 @@ public: //! Return interfaces for accessing wallets (if any). virtual std::vector<std::unique_ptr<Wallet>> getWallets() = 0; + //! Attempts to load a wallet from file or directory. + //! The loaded wallet is also notified to handlers previously registered + //! with handleLoadWallet. + virtual std::unique_ptr<Wallet> loadWallet(const std::string& name, std::string& error, std::string& warning) = 0; + //! Register handler for init messages. using InitMessageFn = std::function<void(const std::string& message)>; virtual std::unique_ptr<Handler> handleInitMessage(InitMessageFn fn) = 0; diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index a2cae2a7a7..7abbee0912 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -56,7 +56,7 @@ public: auto locked_chain = m_wallet.chain().lock(); LOCK(m_wallet.cs_wallet); CValidationState state; - if (!m_wallet.CommitTransaction(m_tx, std::move(value_map), std::move(order_form), m_key, g_connman.get(), state)) { + if (!m_wallet.CommitTransaction(m_tx, std::move(value_map), std::move(order_form), m_key, state)) { reject_reason = state.GetRejectReason(); return false; } @@ -99,7 +99,7 @@ WalletTx MakeWalletTx(interfaces::Chain::Lock& locked_chain, CWallet& wallet, co //! Construct wallet tx status struct. WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx) { - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. + LockAnnotation lock(::cs_main); // Temporary, for mapBlockIndex below. Removed in upcoming commit. WalletTxStatus result; auto mi = ::mapBlockIndex.find(wtx.hashBlock); @@ -109,7 +109,7 @@ WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const C result.depth_in_main_chain = wtx.GetDepthInMainChain(locked_chain); result.time_received = wtx.nTimeReceived; result.lock_time = wtx.tx->nLockTime; - result.is_final = CheckFinalTx(*wtx.tx); + result.is_final = locked_chain.checkFinalTx(*wtx.tx); result.is_trusted = wtx.IsTrusted(locked_chain); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); @@ -135,55 +135,55 @@ WalletTxOut MakeWalletTxOut(interfaces::Chain::Lock& locked_chain, class WalletImpl : public Wallet { public: - explicit WalletImpl(const std::shared_ptr<CWallet>& wallet) : m_shared_wallet(wallet), m_wallet(*wallet.get()) {} + explicit WalletImpl(const std::shared_ptr<CWallet>& wallet) : m_wallet(wallet) {} bool encryptWallet(const SecureString& wallet_passphrase) override { - return m_wallet.EncryptWallet(wallet_passphrase); + return m_wallet->EncryptWallet(wallet_passphrase); } - bool isCrypted() override { return m_wallet.IsCrypted(); } - bool lock() override { return m_wallet.Lock(); } - bool unlock(const SecureString& wallet_passphrase) override { return m_wallet.Unlock(wallet_passphrase); } - bool isLocked() override { return m_wallet.IsLocked(); } + bool isCrypted() override { return m_wallet->IsCrypted(); } + bool lock() override { return m_wallet->Lock(); } + bool unlock(const SecureString& wallet_passphrase) override { return m_wallet->Unlock(wallet_passphrase); } + bool isLocked() override { return m_wallet->IsLocked(); } bool changeWalletPassphrase(const SecureString& old_wallet_passphrase, const SecureString& new_wallet_passphrase) override { - return m_wallet.ChangeWalletPassphrase(old_wallet_passphrase, new_wallet_passphrase); + return m_wallet->ChangeWalletPassphrase(old_wallet_passphrase, new_wallet_passphrase); } - void abortRescan() override { m_wallet.AbortRescan(); } - bool backupWallet(const std::string& filename) override { return m_wallet.BackupWallet(filename); } - std::string getWalletName() override { return m_wallet.GetName(); } + void abortRescan() override { m_wallet->AbortRescan(); } + bool backupWallet(const std::string& filename) override { return m_wallet->BackupWallet(filename); } + std::string getWalletName() override { return m_wallet->GetName(); } bool getKeyFromPool(bool internal, CPubKey& pub_key) override { - return m_wallet.GetKeyFromPool(pub_key, internal); + return m_wallet->GetKeyFromPool(pub_key, internal); } - bool getPubKey(const CKeyID& address, CPubKey& pub_key) override { return m_wallet.GetPubKey(address, pub_key); } - bool getPrivKey(const CKeyID& address, CKey& key) override { return m_wallet.GetKey(address, key); } - bool isSpendable(const CTxDestination& dest) override { return IsMine(m_wallet, dest) & ISMINE_SPENDABLE; } - bool haveWatchOnly() override { return m_wallet.HaveWatchOnly(); }; + bool getPubKey(const CKeyID& address, CPubKey& pub_key) override { return m_wallet->GetPubKey(address, pub_key); } + bool getPrivKey(const CKeyID& address, CKey& key) override { return m_wallet->GetKey(address, key); } + bool isSpendable(const CTxDestination& dest) override { return IsMine(*m_wallet, dest) & ISMINE_SPENDABLE; } + bool haveWatchOnly() override { return m_wallet->HaveWatchOnly(); }; bool setAddressBook(const CTxDestination& dest, const std::string& name, const std::string& purpose) override { - return m_wallet.SetAddressBook(dest, name, purpose); + return m_wallet->SetAddressBook(dest, name, purpose); } bool delAddressBook(const CTxDestination& dest) override { - return m_wallet.DelAddressBook(dest); + return m_wallet->DelAddressBook(dest); } bool getAddress(const CTxDestination& dest, std::string* name, isminetype* is_mine, std::string* purpose) override { - LOCK(m_wallet.cs_wallet); - auto it = m_wallet.mapAddressBook.find(dest); - if (it == m_wallet.mapAddressBook.end()) { + LOCK(m_wallet->cs_wallet); + auto it = m_wallet->mapAddressBook.find(dest); + if (it == m_wallet->mapAddressBook.end()) { return false; } if (name) { *name = it->second.name; } if (is_mine) { - *is_mine = IsMine(m_wallet, dest); + *is_mine = IsMine(*m_wallet, dest); } if (purpose) { *purpose = it->second.purpose; @@ -192,51 +192,52 @@ public: } std::vector<WalletAddress> getAddresses() override { - LOCK(m_wallet.cs_wallet); + LOCK(m_wallet->cs_wallet); std::vector<WalletAddress> result; - for (const auto& item : m_wallet.mapAddressBook) { - result.emplace_back(item.first, IsMine(m_wallet, item.first), item.second.name, item.second.purpose); + for (const auto& item : m_wallet->mapAddressBook) { + result.emplace_back(item.first, IsMine(*m_wallet, item.first), item.second.name, item.second.purpose); } return result; } - void learnRelatedScripts(const CPubKey& key, OutputType type) override { m_wallet.LearnRelatedScripts(key, type); } + void learnRelatedScripts(const CPubKey& key, OutputType type) override { m_wallet->LearnRelatedScripts(key, type); } bool addDestData(const CTxDestination& dest, const std::string& key, const std::string& value) override { - LOCK(m_wallet.cs_wallet); - return m_wallet.AddDestData(dest, key, value); + LOCK(m_wallet->cs_wallet); + return m_wallet->AddDestData(dest, key, value); } bool eraseDestData(const CTxDestination& dest, const std::string& key) override { - LOCK(m_wallet.cs_wallet); - return m_wallet.EraseDestData(dest, key); + LOCK(m_wallet->cs_wallet); + return m_wallet->EraseDestData(dest, key); } std::vector<std::string> getDestValues(const std::string& prefix) override { - return m_wallet.GetDestValues(prefix); + LOCK(m_wallet->cs_wallet); + return m_wallet->GetDestValues(prefix); } void lockCoin(const COutPoint& output) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.LockCoin(output); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->LockCoin(output); } void unlockCoin(const COutPoint& output) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.UnlockCoin(output); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->UnlockCoin(output); } bool isLockedCoin(const COutPoint& output) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.IsLockedCoin(output.hash, output.n); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->IsLockedCoin(output.hash, output.n); } void listLockedCoins(std::vector<COutPoint>& outputs) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.ListLockedCoins(outputs); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->ListLockedCoins(outputs); } std::unique_ptr<PendingWalletTx> createTransaction(const std::vector<CRecipient>& recipients, const CCoinControl& coin_control, @@ -245,25 +246,25 @@ public: CAmount& fee, std::string& fail_reason) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - auto pending = MakeUnique<PendingWalletTxImpl>(m_wallet); - if (!m_wallet.CreateTransaction(*locked_chain, recipients, pending->m_tx, pending->m_key, fee, change_pos, + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + auto pending = MakeUnique<PendingWalletTxImpl>(*m_wallet); + if (!m_wallet->CreateTransaction(*locked_chain, recipients, pending->m_tx, pending->m_key, fee, change_pos, fail_reason, coin_control, sign)) { return {}; } return std::move(pending); } - bool transactionCanBeAbandoned(const uint256& txid) override { return m_wallet.TransactionCanBeAbandoned(txid); } + bool transactionCanBeAbandoned(const uint256& txid) override { return m_wallet->TransactionCanBeAbandoned(txid); } bool abandonTransaction(const uint256& txid) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.AbandonTransaction(*locked_chain, txid); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->AbandonTransaction(*locked_chain, txid); } bool transactionCanBeBumped(const uint256& txid) override { - return feebumper::TransactionCanBeBumped(&m_wallet, txid); + return feebumper::TransactionCanBeBumped(m_wallet.get(), txid); } bool createBumpTransaction(const uint256& txid, const CCoinControl& coin_control, @@ -273,46 +274,46 @@ public: CAmount& new_fee, CMutableTransaction& mtx) override { - return feebumper::CreateTransaction(&m_wallet, txid, coin_control, total_fee, errors, old_fee, new_fee, mtx) == + return feebumper::CreateTransaction(m_wallet.get(), txid, coin_control, total_fee, errors, old_fee, new_fee, mtx) == feebumper::Result::OK; } - bool signBumpTransaction(CMutableTransaction& mtx) override { return feebumper::SignTransaction(&m_wallet, mtx); } + bool signBumpTransaction(CMutableTransaction& mtx) override { return feebumper::SignTransaction(m_wallet.get(), mtx); } bool commitBumpTransaction(const uint256& txid, CMutableTransaction&& mtx, std::vector<std::string>& errors, uint256& bumped_txid) override { - return feebumper::CommitTransaction(&m_wallet, txid, std::move(mtx), errors, bumped_txid) == + return feebumper::CommitTransaction(m_wallet.get(), txid, std::move(mtx), errors, bumped_txid) == feebumper::Result::OK; } CTransactionRef getTx(const uint256& txid) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - auto mi = m_wallet.mapWallet.find(txid); - if (mi != m_wallet.mapWallet.end()) { + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + auto mi = m_wallet->mapWallet.find(txid); + if (mi != m_wallet->mapWallet.end()) { return mi->second.tx; } return {}; } WalletTx getWalletTx(const uint256& txid) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - auto mi = m_wallet.mapWallet.find(txid); - if (mi != m_wallet.mapWallet.end()) { - return MakeWalletTx(*locked_chain, m_wallet, mi->second); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + auto mi = m_wallet->mapWallet.find(txid); + if (mi != m_wallet->mapWallet.end()) { + return MakeWalletTx(*locked_chain, *m_wallet, mi->second); } return {}; } std::vector<WalletTx> getWalletTxs() override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); std::vector<WalletTx> result; - result.reserve(m_wallet.mapWallet.size()); - for (const auto& entry : m_wallet.mapWallet) { - result.emplace_back(MakeWalletTx(*locked_chain, m_wallet, entry.second)); + result.reserve(m_wallet->mapWallet.size()); + for (const auto& entry : m_wallet->mapWallet) { + result.emplace_back(MakeWalletTx(*locked_chain, *m_wallet, entry.second)); } return result; } @@ -321,16 +322,16 @@ public: int& num_blocks, int64_t& block_time) override { - auto locked_chain = m_wallet.chain().lock(true /* try_lock */); + auto locked_chain = m_wallet->chain().lock(true /* try_lock */); if (!locked_chain) { return false; } - TRY_LOCK(m_wallet.cs_wallet, locked_wallet); + TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; } - auto mi = m_wallet.mapWallet.find(txid); - if (mi == m_wallet.mapWallet.end()) { + auto mi = m_wallet->mapWallet.find(txid); + if (mi == m_wallet->mapWallet.end()) { return false; } if (Optional<int> height = locked_chain->getHeight()) { @@ -349,37 +350,37 @@ public: bool& in_mempool, int& num_blocks) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - auto mi = m_wallet.mapWallet.find(txid); - if (mi != m_wallet.mapWallet.end()) { + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + auto mi = m_wallet->mapWallet.find(txid); + if (mi != m_wallet->mapWallet.end()) { num_blocks = locked_chain->getHeight().get_value_or(-1); in_mempool = mi->second.InMempool(); order_form = mi->second.vOrderForm; tx_status = MakeWalletTxStatus(*locked_chain, mi->second); - return MakeWalletTx(*locked_chain, m_wallet, mi->second); + return MakeWalletTx(*locked_chain, *m_wallet, mi->second); } return {}; } WalletBalances getBalances() override { WalletBalances result; - result.balance = m_wallet.GetBalance(); - result.unconfirmed_balance = m_wallet.GetUnconfirmedBalance(); - result.immature_balance = m_wallet.GetImmatureBalance(); - result.have_watch_only = m_wallet.HaveWatchOnly(); + result.balance = m_wallet->GetBalance(); + result.unconfirmed_balance = m_wallet->GetUnconfirmedBalance(); + result.immature_balance = m_wallet->GetImmatureBalance(); + result.have_watch_only = m_wallet->HaveWatchOnly(); if (result.have_watch_only) { - result.watch_only_balance = m_wallet.GetBalance(ISMINE_WATCH_ONLY); - result.unconfirmed_watch_only_balance = m_wallet.GetUnconfirmedWatchOnlyBalance(); - result.immature_watch_only_balance = m_wallet.GetImmatureWatchOnlyBalance(); + result.watch_only_balance = m_wallet->GetBalance(ISMINE_WATCH_ONLY); + result.unconfirmed_watch_only_balance = m_wallet->GetUnconfirmedWatchOnlyBalance(); + result.immature_watch_only_balance = m_wallet->GetImmatureWatchOnlyBalance(); } return result; } bool tryGetBalances(WalletBalances& balances, int& num_blocks) override { - auto locked_chain = m_wallet.chain().lock(true /* try_lock */); + auto locked_chain = m_wallet->chain().lock(true /* try_lock */); if (!locked_chain) return false; - TRY_LOCK(m_wallet.cs_wallet, locked_wallet); + TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; } @@ -387,68 +388,68 @@ public: num_blocks = locked_chain->getHeight().get_value_or(-1); return true; } - CAmount getBalance() override { return m_wallet.GetBalance(); } + CAmount getBalance() override { return m_wallet->GetBalance(); } CAmount getAvailableBalance(const CCoinControl& coin_control) override { - return m_wallet.GetAvailableBalance(&coin_control); + return m_wallet->GetAvailableBalance(&coin_control); } isminetype txinIsMine(const CTxIn& txin) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.IsMine(txin); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->IsMine(txin); } isminetype txoutIsMine(const CTxOut& txout) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.IsMine(txout); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->IsMine(txout); } CAmount getDebit(const CTxIn& txin, isminefilter filter) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.GetDebit(txin, filter); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->GetDebit(txin, filter); } CAmount getCredit(const CTxOut& txout, isminefilter filter) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); - return m_wallet.GetCredit(txout, filter); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); + return m_wallet->GetCredit(txout, filter); } CoinsList listCoins() override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); CoinsList result; - for (const auto& entry : m_wallet.ListCoins(*locked_chain)) { + for (const auto& entry : m_wallet->ListCoins(*locked_chain)) { auto& group = result[entry.first]; for (const auto& coin : entry.second) { group.emplace_back(COutPoint(coin.tx->GetHash(), coin.i), - MakeWalletTxOut(*locked_chain, m_wallet, *coin.tx, coin.i, coin.nDepth)); + MakeWalletTxOut(*locked_chain, *m_wallet, *coin.tx, coin.i, coin.nDepth)); } } return result; } std::vector<WalletTxOut> getCoins(const std::vector<COutPoint>& outputs) override { - auto locked_chain = m_wallet.chain().lock(); - LOCK(m_wallet.cs_wallet); + auto locked_chain = m_wallet->chain().lock(); + LOCK(m_wallet->cs_wallet); std::vector<WalletTxOut> result; result.reserve(outputs.size()); for (const auto& output : outputs) { result.emplace_back(); - auto it = m_wallet.mapWallet.find(output.hash); - if (it != m_wallet.mapWallet.end()) { + auto it = m_wallet->mapWallet.find(output.hash); + if (it != m_wallet->mapWallet.end()) { int depth = it->second.GetDepthInMainChain(*locked_chain); if (depth >= 0) { - result.back() = MakeWalletTxOut(*locked_chain, m_wallet, it->second, output.n, depth); + result.back() = MakeWalletTxOut(*locked_chain, *m_wallet, it->second, output.n, depth); } } } return result; } - CAmount getRequiredFee(unsigned int tx_bytes) override { return GetRequiredFee(m_wallet, tx_bytes); } + CAmount getRequiredFee(unsigned int tx_bytes) override { return GetRequiredFee(*m_wallet, tx_bytes); } CAmount getMinimumFee(unsigned int tx_bytes, const CCoinControl& coin_control, int* returned_target, @@ -456,50 +457,54 @@ public: { FeeCalculation fee_calc; CAmount result; - result = GetMinimumFee(m_wallet, tx_bytes, coin_control, ::mempool, ::feeEstimator, &fee_calc); + result = GetMinimumFee(*m_wallet, tx_bytes, coin_control, &fee_calc); if (returned_target) *returned_target = fee_calc.returnedTarget; if (reason) *reason = fee_calc.reason; return result; } - unsigned int getConfirmTarget() override { return m_wallet.m_confirm_target; } - bool hdEnabled() override { return m_wallet.IsHDEnabled(); } - bool IsWalletFlagSet(uint64_t flag) override { return m_wallet.IsWalletFlagSet(flag); } - OutputType getDefaultAddressType() override { return m_wallet.m_default_address_type; } - OutputType getDefaultChangeType() override { return m_wallet.m_default_change_type; } + unsigned int getConfirmTarget() override { return m_wallet->m_confirm_target; } + bool hdEnabled() override { return m_wallet->IsHDEnabled(); } + bool canGetAddresses() override { return m_wallet->CanGetAddresses(); } + bool IsWalletFlagSet(uint64_t flag) override { return m_wallet->IsWalletFlagSet(flag); } + OutputType getDefaultAddressType() override { return m_wallet->m_default_address_type; } + OutputType getDefaultChangeType() override { return m_wallet->m_default_change_type; } + void remove() override + { + RemoveWallet(m_wallet); + } std::unique_ptr<Handler> handleUnload(UnloadFn fn) override { - return MakeHandler(m_wallet.NotifyUnload.connect(fn)); + return MakeHandler(m_wallet->NotifyUnload.connect(fn)); } std::unique_ptr<Handler> handleShowProgress(ShowProgressFn fn) override { - return MakeHandler(m_wallet.ShowProgress.connect(fn)); + return MakeHandler(m_wallet->ShowProgress.connect(fn)); } std::unique_ptr<Handler> handleStatusChanged(StatusChangedFn fn) override { - return MakeHandler(m_wallet.NotifyStatusChanged.connect([fn](CCryptoKeyStore*) { fn(); })); + return MakeHandler(m_wallet->NotifyStatusChanged.connect([fn](CCryptoKeyStore*) { fn(); })); } std::unique_ptr<Handler> handleAddressBookChanged(AddressBookChangedFn fn) override { - return MakeHandler(m_wallet.NotifyAddressBookChanged.connect( + return MakeHandler(m_wallet->NotifyAddressBookChanged.connect( [fn](CWallet*, const CTxDestination& address, const std::string& label, bool is_mine, const std::string& purpose, ChangeType status) { fn(address, label, is_mine, purpose, status); })); } std::unique_ptr<Handler> handleTransactionChanged(TransactionChangedFn fn) override { - return MakeHandler(m_wallet.NotifyTransactionChanged.connect( + return MakeHandler(m_wallet->NotifyTransactionChanged.connect( [fn](CWallet*, const uint256& txid, ChangeType status) { fn(txid, status); })); } std::unique_ptr<Handler> handleWatchOnlyChanged(WatchOnlyChangedFn fn) override { - return MakeHandler(m_wallet.NotifyWatchonlyChanged.connect(fn)); + return MakeHandler(m_wallet->NotifyWatchonlyChanged.connect(fn)); } std::unique_ptr<Handler> handleCanGetAddressesChanged(CanGetAddressesChangedFn fn) override { - return MakeHandler(m_wallet.NotifyCanGetAddressesChanged.connect(fn)); + return MakeHandler(m_wallet->NotifyCanGetAddressesChanged.connect(fn)); } - std::shared_ptr<CWallet> m_shared_wallet; - CWallet& m_wallet; + std::shared_ptr<CWallet> m_wallet; }; class WalletClientImpl : public ChainClient @@ -523,7 +528,7 @@ public: } // namespace -std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) { return MakeUnique<WalletImpl>(wallet); } +std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) { return wallet ? MakeUnique<WalletImpl>(wallet) : nullptr; } std::unique_ptr<ChainClient> MakeWalletClient(Chain& chain, std::vector<std::string> wallet_filenames) { diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 72c64ded01..a931e5fafb 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -235,6 +235,9 @@ public: // Return whether HD enabled. virtual bool hdEnabled() = 0; + // Return whether the wallet is blank. + virtual bool canGetAddresses() = 0; + // check if a certain wallet flag is set. virtual bool IsWalletFlagSet(uint64_t flag) = 0; @@ -244,6 +247,9 @@ public: // Get default change type. virtual OutputType getDefaultChangeType() = 0; + // Remove wallet. + virtual void remove() = 0; + //! Register handler for unload message. using UnloadFn = std::function<void()>; virtual std::unique_ptr<Handler> handleUnload(UnloadFn fn) = 0; diff --git a/src/key_io.cpp b/src/key_io.cpp index d998089535..1d53a5e074 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -142,7 +142,9 @@ CKey DecodeSecret(const std::string& str) key.Set(data.begin() + privkey_prefix.size(), data.begin() + privkey_prefix.size() + 32, compressed); } } - memory_cleanse(data.data(), data.size()); + if (!data.empty()) { + memory_cleanse(data.data(), data.size()); + } return key; } diff --git a/src/miner.cpp b/src/miner.cpp index ef48a86e32..80a2f8f018 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,8 +10,8 @@ #include <chainparams.h> #include <coins.h> #include <consensus/consensus.h> -#include <consensus/tx_verify.h> #include <consensus/merkle.h> +#include <consensus/tx_verify.h> #include <consensus/validation.h> #include <hash.h> #include <net.h> @@ -21,22 +21,14 @@ #include <primitives/transaction.h> #include <script/standard.h> #include <timedata.h> -#include <util/system.h> #include <util/moneystr.h> +#include <util/system.h> #include <validationinterface.h> #include <algorithm> #include <queue> #include <utility> -// Unconfirmed transactions in the memory pool often depend on other -// transactions in the memory pool. When we select transactions from the -// pool, we select by highest fee rate of a transaction combined with all -// its ancestors. - -uint64_t nLastBlockTx = 0; -uint64_t nLastBlockWeight = 0; - int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { int64_t nOldTime = pblock->nTime; @@ -95,6 +87,9 @@ void BlockAssembler::resetBlock() nFees = 0; } +Optional<int64_t> BlockAssembler::m_last_block_num_txs{nullopt}; +Optional<int64_t> BlockAssembler::m_last_block_weight{nullopt}; + std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& scriptPubKeyIn) { int64_t nTimeStart = GetTimeMicros(); @@ -147,8 +142,8 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc int64_t nTime1 = GetTimeMicros(); - nLastBlockTx = nBlockTx; - nLastBlockWeight = nBlockWeight; + m_last_block_num_txs = nBlockTx; + m_last_block_weight = nBlockWeight; // Create coinbase transaction. CMutableTransaction coinbaseTx; diff --git a/src/miner.h b/src/miner.h index 44c50b01ad..7c4c455072 100644 --- a/src/miner.h +++ b/src/miner.h @@ -1,17 +1,19 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_MINER_H #define BITCOIN_MINER_H +#include <optional.h> #include <primitives/block.h> #include <txmempool.h> #include <validation.h> -#include <stdint.h> #include <memory> +#include <stdint.h> + #include <boost/multi_index_container.hpp> #include <boost/multi_index/ordered_index.hpp> @@ -159,6 +161,9 @@ public: /** Construct a new block template with coinbase to scriptPubKeyIn */ std::unique_ptr<CBlockTemplate> CreateNewBlock(const CScript& scriptPubKeyIn); + static Optional<int64_t> m_last_block_num_txs; + static Optional<int64_t> m_last_block_weight; + private: // utility functions /** Clear the block's state and prepare for assembling a new block */ diff --git a/src/net.cpp b/src/net.cpp index d52d5b7cf5..ccab4a1718 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -85,8 +85,6 @@ std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(cs_mapLocalHost); static bool vfLimited[NET_MAX] GUARDED_BY(cs_mapLocalHost) = {}; std::string strSubVersion; -limitedmap<uint256, int64_t> mapAlreadyAskedFor(MAX_INV_SZ); - void CConnman::AddOneShot(const std::string& strDest) { LOCK(cs_vOneShots); @@ -1767,9 +1765,15 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) addr = addrman.Select(fFeeler); } - // if we selected an invalid address, restart - if (!addr.IsValid() || setConnected.count(addr.GetGroup()) || IsLocal(addr)) + // Require outbound connections, other than feelers, to be to distinct network groups + if (!fFeeler && setConnected.count(addr.GetGroup())) { break; + } + + // if we selected an invalid or local address, restart + if (!addr.IsValid() || IsLocal(addr)) { + break; + } // If we didn't find an appropriate destination after trying 100 addresses fetched from addrman, // stop this loop, and let the outer loop run again (which sleeps, adds seed nodes, recalculates @@ -2644,40 +2648,6 @@ CNode::~CNode() CloseSocket(hSocket); } -void CNode::AskFor(const CInv& inv) -{ - if (mapAskFor.size() > MAPASKFOR_MAX_SZ || setAskFor.size() > SETASKFOR_MAX_SZ) - return; - // a peer may not have multiple non-responded queue positions for a single inv item - if (!setAskFor.insert(inv.hash).second) - return; - - // We're using mapAskFor as a priority queue, - // the key is the earliest time the request can be sent - int64_t nRequestTime; - limitedmap<uint256, int64_t>::const_iterator it = mapAlreadyAskedFor.find(inv.hash); - if (it != mapAlreadyAskedFor.end()) - nRequestTime = it->second; - else - nRequestTime = 0; - LogPrint(BCLog::NET, "askfor %s %d (%s) peer=%d\n", inv.ToString(), nRequestTime, FormatISO8601Time(nRequestTime/1000000), id); - - // Make sure not to reuse time indexes to keep things in the same order - int64_t nNow = GetTimeMicros() - 1000000; - static int64_t nLastTime; - ++nLastTime; - nNow = std::max(nNow, nLastTime); - nLastTime = nNow; - - // Each retry is 2 minutes after the last - nRequestTime = std::max(nRequestTime + 2 * 60 * 1000000, nNow); - if (it != mapAlreadyAskedFor.end()) - mapAlreadyAskedFor.update(it, nRequestTime); - else - mapAlreadyAskedFor.insert(std::make_pair(inv.hash, nRequestTime)); - mapAskFor.insert(std::make_pair(nRequestTime, inv)); -} - bool CConnman::NodeFullyConnected(const CNode* pnode) { return pnode && pnode->fSuccessfullyConnected && !pnode->fDisconnect; @@ -67,10 +67,6 @@ static const bool DEFAULT_UPNP = USE_UPNP; #else static const bool DEFAULT_UPNP = false; #endif -/** The maximum number of entries in mapAskFor */ -static const size_t MAPASKFOR_MAX_SZ = MAX_INV_SZ; -/** The maximum number of entries in setAskFor (larger due to getdata latency)*/ -static const size_t SETASKFOR_MAX_SZ = 2 * MAX_INV_SZ; /** The maximum number of peer connections to maintain. */ static const unsigned int DEFAULT_MAX_PEER_CONNECTIONS = 125; /** The default for -maxuploadtarget. 0 = Unlimited */ @@ -178,7 +174,18 @@ public: CConnman(uint64_t seed0, uint64_t seed1); ~CConnman(); bool Start(CScheduler& scheduler, const Options& options); - void Stop(); + + // TODO: Remove NO_THREAD_SAFETY_ANALYSIS. Lock cs_vNodes before reading the variable vNodes. + // + // When removing NO_THREAD_SAFETY_ANALYSIS be aware of the following lock order requirements: + // * CheckForStaleTipAndEvictPeers locks cs_main before indirectly calling GetExtraOutboundCount + // which locks cs_vNodes. + // * ProcessMessage locks cs_main and g_cs_orphans before indirectly calling ForEachNode which + // locks cs_vNodes. + // + // Thus the implicit locking order requirement is: (1) cs_main, (2) g_cs_orphans, (3) cs_vNodes. + void Stop() NO_THREAD_SAFETY_ANALYSIS; + void Interrupt(); bool GetNetworkActive() const { return fNetworkActive; }; bool GetUseAddrmanOutgoing() const { return m_use_addrman_outgoing; }; @@ -386,7 +393,7 @@ private: CCriticalSection cs_vOneShots; std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); CCriticalSection cs_vAddedNodes; - std::vector<CNode*> vNodes; + std::vector<CNode*> vNodes GUARDED_BY(cs_vNodes); std::list<CNode*> vNodesDisconnected; mutable CCriticalSection cs_vNodes; std::atomic<NodeId> nLastNodeId{0}; @@ -514,8 +521,6 @@ extern bool fDiscover; extern bool fListen; extern bool fRelayTxes; -extern limitedmap<uint256, int64_t> mapAlreadyAskedFor; - /** Subversion as sent to the P2P network in `version` messages */ extern std::string strSubVersion; @@ -704,8 +709,6 @@ public: // and in the order requested. std::vector<uint256> vInventoryBlockToSend GUARDED_BY(cs_inventory); CCriticalSection cs_inventory; - std::set<uint256> setAskFor; - std::multimap<int64_t, CInv> mapAskFor; int64_t nNextInvSend{0}; // Used for headers announcements - unfiltered blocks to relay std::vector<uint256> vBlockHashesToAnnounce GUARDED_BY(cs_inventory); @@ -852,8 +855,6 @@ public: vBlockHashesToAnnounce.push_back(hash); } - void AskFor(const CInv& inv); - void CloseSocketDisconnect(); void copyStats(CNodeStats &stats); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 62b7d4e966..18dd2f010c 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -64,12 +64,28 @@ static constexpr int STALE_RELAY_AGE_LIMIT = 30 * 24 * 60 * 60; /// Age after which a block is considered historical for purposes of rate /// limiting block relay. Set to one week, denominated in seconds. static constexpr int HISTORICAL_BLOCK_AGE = 7 * 24 * 60 * 60; +/** Maximum number of in-flight transactions from a peer */ +static constexpr int32_t MAX_PEER_TX_IN_FLIGHT = 100; +/** Maximum number of announced transactions from a peer */ +static constexpr int32_t MAX_PEER_TX_ANNOUNCEMENTS = 2 * MAX_INV_SZ; +/** How many microseconds to delay requesting transactions from inbound peers */ +static constexpr int64_t INBOUND_PEER_TX_DELAY = 2 * 1000000; +/** How long to wait (in microseconds) before downloading a transaction from an additional peer */ +static constexpr int64_t GETDATA_TX_INTERVAL = 60 * 1000000; +/** Maximum delay (in microseconds) for transaction requests to avoid biasing some peers over others. */ +static constexpr int64_t MAX_GETDATA_RANDOM_DELAY = 2 * 1000000; +static_assert(INBOUND_PEER_TX_DELAY >= MAX_GETDATA_RANDOM_DELAY, +"To preserve security, MAX_GETDATA_RANDOM_DELAY should not exceed INBOUND_PEER_DELAY"); +/** Limit to avoid sending big packets. Not used in processing incoming GETDATA for compatibility */ +static const unsigned int MAX_GETDATA_SZ = 1000; + struct COrphanTx { // When modifying, adapt the copy of this definition in tests/DoS_tests. CTransactionRef tx; NodeId fromPeer; int64_t nTimeExpire; + size_t list_pos; }; CCriticalSection g_cs_orphans; std::map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(g_cs_orphans); @@ -171,6 +187,8 @@ namespace { }; std::map<COutPoint, std::set<std::map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(g_cs_orphans); + std::vector<std::map<uint256, COrphanTx>::iterator> g_orphan_list GUARDED_BY(g_cs_orphans); //! For random eviction + static size_t vExtraTxnForCompactIt GUARDED_BY(g_cs_orphans) = 0; static std::vector<std::pair<uint256, CTransactionRef>> vExtraTxnForCompact GUARDED_BY(g_cs_orphans); } // namespace @@ -274,6 +292,66 @@ struct CNodeState { //! Time of last new block announcement int64_t m_last_block_announcement; + /* + * State associated with transaction download. + * + * Tx download algorithm: + * + * When inv comes in, queue up (process_time, txid) inside the peer's + * CNodeState (m_tx_process_time) as long as m_tx_announced for the peer + * isn't too big (MAX_PEER_TX_ANNOUNCEMENTS). + * + * The process_time for a transaction is set to nNow for outbound peers, + * nNow + 2 seconds for inbound peers. This is the time at which we'll + * consider trying to request the transaction from the peer in + * SendMessages(). The delay for inbound peers is to allow outbound peers + * a chance to announce before we request from inbound peers, to prevent + * an adversary from using inbound connections to blind us to a + * transaction (InvBlock). + * + * When we call SendMessages() for a given peer, + * we will loop over the transactions in m_tx_process_time, looking + * at the transactions whose process_time <= nNow. We'll request each + * such transaction that we don't have already and that hasn't been + * requested from another peer recently, up until we hit the + * MAX_PEER_TX_IN_FLIGHT limit for the peer. Then we'll update + * g_already_asked_for for each requested txid, storing the time of the + * GETDATA request. We use g_already_asked_for to coordinate transaction + * requests amongst our peers. + * + * For transactions that we still need but we have already recently + * requested from some other peer, we'll reinsert (process_time, txid) + * back into the peer's m_tx_process_time at the point in the future at + * which the most recent GETDATA request would time out (ie + * GETDATA_TX_INTERVAL + the request time stored in g_already_asked_for). + * We add an additional delay for inbound peers, again to prefer + * attempting download from outbound peers first. + * We also add an extra small random delay up to 2 seconds + * to avoid biasing some peers over others. (e.g., due to fixed ordering + * of peer processing in ThreadMessageHandler). + * + * When we receive a transaction from a peer, we remove the txid from the + * peer's m_tx_in_flight set and from their recently announced set + * (m_tx_announced). We also clear g_already_asked_for for that entry, so + * that if somehow the transaction is not accepted but also not added to + * the reject filter, then we will eventually redownload from other + * peers. + */ + struct TxDownloadState { + /* Track when to attempt download of announced transactions (process + * time in micros -> txid) + */ + std::multimap<int64_t, uint256> m_tx_process_time; + + //! Store all the transactions a peer has recently announced + std::set<uint256> m_tx_announced; + + //! Store transactions which were requested by us + std::set<uint256> m_tx_in_flight; + }; + + TxDownloadState m_tx_download; + CNodeState(CAddress addrIn, std::string addrNameIn) : address(addrIn), name(addrNameIn) { fCurrentlyConnected = false; nMisbehavior = 0; @@ -301,6 +379,9 @@ struct CNodeState { } }; +// Keeps track of the time (in microseconds) when transactions were requested last time +limitedmap<uint256, int64_t> g_already_asked_for GUARDED_BY(cs_main)(MAX_INV_SZ); + /** Map maintaining per-node state. */ static std::map<NodeId, CNodeState> mapNodeState GUARDED_BY(cs_main); @@ -591,6 +672,58 @@ static void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vec } } +void EraseTxRequest(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + g_already_asked_for.erase(txid); +} + +int64_t GetTxRequestTime(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + auto it = g_already_asked_for.find(txid); + if (it != g_already_asked_for.end()) { + return it->second; + } + return 0; +} + +void UpdateTxRequestTime(const uint256& txid, int64_t request_time) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + auto it = g_already_asked_for.find(txid); + if (it == g_already_asked_for.end()) { + g_already_asked_for.insert(std::make_pair(txid, request_time)); + } else { + g_already_asked_for.update(it, request_time); + } +} + + +void RequestTx(CNodeState* state, const uint256& txid, int64_t nNow) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +{ + CNodeState::TxDownloadState& peer_download_state = state->m_tx_download; + if (peer_download_state.m_tx_announced.size() >= MAX_PEER_TX_ANNOUNCEMENTS || peer_download_state.m_tx_announced.count(txid)) { + // Too many queued announcements from this peer, or we already have + // this announcement + return; + } + peer_download_state.m_tx_announced.insert(txid); + + int64_t process_time; + int64_t last_request_time = GetTxRequestTime(txid); + // First time requesting this tx + if (last_request_time == 0) { + process_time = nNow; + } else { + // Randomize the delay to avoid biasing some peers over others (such as due to + // fixed ordering of peer processing in ThreadMessageHandler) + process_time = last_request_time + GETDATA_TX_INTERVAL + GetRand(MAX_GETDATA_RANDOM_DELAY); + } + + // We delay processing announcements from non-preferred (eg inbound) peers + if (!state->fPreferredDownload) process_time += INBOUND_PEER_TX_DELAY; + + peer_download_state.m_tx_process_time.emplace(process_time, txid); +} + } // namespace // This function is used for testing the stale tip eviction logic, see @@ -707,8 +840,9 @@ bool AddOrphanTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRE return false; } - auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME}); + auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME, g_orphan_list.size()}); assert(ret.second); + g_orphan_list.push_back(ret.first); for (const CTxIn& txin : tx->vin) { mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); } @@ -734,6 +868,18 @@ int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) if (itPrev->second.empty()) mapOrphanTransactionsByPrev.erase(itPrev); } + + size_t old_pos = it->second.list_pos; + assert(g_orphan_list[old_pos] == it); + if (old_pos + 1 != g_orphan_list.size()) { + // Unless we're deleting the last entry in g_orphan_list, move the last + // entry to the position we're deleting. + auto it_last = g_orphan_list.back(); + g_orphan_list[old_pos] = it_last; + it_last->second.list_pos = old_pos; + } + g_orphan_list.pop_back(); + mapOrphanTransactions.erase(it); return 1; } @@ -784,11 +930,8 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) while (mapOrphanTransactions.size() > nMaxOrphans) { // Evict a random orphan: - uint256 randomhash = rng.rand256(); - std::map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.lower_bound(randomhash); - if (it == mapOrphanTransactions.end()) - it = mapOrphanTransactions.begin(); - EraseOrphanTx(it->first); + size_t randompos = rng.randrange(g_orphan_list.size()); + EraseOrphanTx(g_orphan_list[randompos]->first); ++nEvicted; } return nEvicted; @@ -1945,6 +2088,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr LOCK(cs_main); uint32_t nFetchFlags = GetFetchFlags(pfrom); + int64_t nNow = GetTimeMicros(); for (CInv &inv : vInv) { @@ -1976,7 +2120,7 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr if (fBlocksOnly) { LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->GetId()); } else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) { - pfrom->AskFor(inv); + RequestTx(State(pfrom->GetId()), inv.hash, nNow); } } } @@ -2211,8 +2355,10 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr bool fMissingInputs = false; CValidationState state; - pfrom->setAskFor.erase(inv.hash); - mapAlreadyAskedFor.erase(inv.hash); + CNodeState* nodestate = State(pfrom->GetId()); + nodestate->m_tx_download.m_tx_announced.erase(inv.hash); + nodestate->m_tx_download.m_tx_in_flight.erase(inv.hash); + EraseTxRequest(inv.hash); std::list<CTransactionRef> lRemovedTxn; @@ -2303,10 +2449,12 @@ bool static ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStr } if (!fRejectedParents) { uint32_t nFetchFlags = GetFetchFlags(pfrom); + int64_t nNow = GetTimeMicros(); + for (const CTxIn& txin : tx.vin) { CInv _inv(MSG_TX | nFetchFlags, txin.prevout.hash); pfrom->AddInventoryKnown(_inv); - if (!AlreadyHave(_inv)) pfrom->AskFor(_inv); + if (!AlreadyHave(_inv)) RequestTx(State(pfrom->GetId()), _inv.hash, nNow); } AddOrphanTx(ptx, pfrom->GetId()); @@ -3731,24 +3879,39 @@ bool PeerLogicValidation::SendMessages(CNode* pto) // // Message: getdata (non-blocks) // - while (!pto->mapAskFor.empty() && (*pto->mapAskFor.begin()).first <= nNow) - { - const CInv& inv = (*pto->mapAskFor.begin()).second; - if (!AlreadyHave(inv)) - { - LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->GetId()); - vGetData.push_back(inv); - if (vGetData.size() >= 1000) - { - connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - vGetData.clear(); + auto& tx_process_time = state.m_tx_download.m_tx_process_time; + while (!tx_process_time.empty() && tx_process_time.begin()->first <= nNow && state.m_tx_download.m_tx_in_flight.size() < MAX_PEER_TX_IN_FLIGHT) { + const uint256& txid = tx_process_time.begin()->second; + CInv inv(MSG_TX | GetFetchFlags(pto), txid); + if (!AlreadyHave(inv)) { + // If this transaction was last requested more than 1 minute ago, + // then request. + int64_t last_request_time = GetTxRequestTime(inv.hash); + if (last_request_time <= nNow - GETDATA_TX_INTERVAL) { + LogPrint(BCLog::NET, "Requesting %s peer=%d\n", inv.ToString(), pto->GetId()); + vGetData.push_back(inv); + if (vGetData.size() >= MAX_GETDATA_SZ) { + connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + vGetData.clear(); + } + UpdateTxRequestTime(inv.hash, nNow); + state.m_tx_download.m_tx_in_flight.insert(inv.hash); + } else { + // This transaction is in flight from someone else; queue + // up processing to happen after the download times out + // (with a slight delay for inbound peers, to prefer + // requests to outbound peers). + RequestTx(&state, txid, nNow); } } else { - //If we're not going to ask, don't expect a response. - pto->setAskFor.erase(inv.hash); + // We have already seen this transaction, no need to download. + state.m_tx_download.m_tx_announced.erase(inv.hash); + state.m_tx_download.m_tx_in_flight.erase(inv.hash); } - pto->mapAskFor.erase(pto->mapAskFor.begin()); + tx_process_time.erase(tx_process_time.begin()); } + + if (!vGetData.empty()) connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); diff --git a/src/netaddress.cpp b/src/netaddress.cpp index a0c7f8e3c2..58e45c2c02 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -182,16 +182,16 @@ bool CNetAddr::IsTor() const bool CNetAddr::IsLocal() const { - // IPv4 loopback - if (IsIPv4() && (GetByte(3) == 127 || GetByte(3) == 0)) - return true; + // IPv4 loopback (127.0.0.0/8 or 0.0.0.0/8) + if (IsIPv4() && (GetByte(3) == 127 || GetByte(3) == 0)) + return true; - // IPv6 loopback (::1/128) - static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; - if (memcmp(ip, pchLocal, 16) == 0) - return true; + // IPv6 loopback (::1/128) + static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; + if (memcmp(ip, pchLocal, 16) == 0) + return true; - return false; + return false; } bool CNetAddr::IsValid() const diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp new file mode 100644 index 0000000000..7b9b4310e7 --- /dev/null +++ b/src/node/transaction.cpp @@ -0,0 +1,103 @@ +// Copyright (c) 2010 Satoshi Nakamoto +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/validation.h> +#include <net.h> +#include <txmempool.h> +#include <validation.h> +#include <validationinterface.h> +#include <node/transaction.h> + +#include <future> + +std::string TransactionErrorString(const TransactionError err) +{ + switch (err) { + case TransactionError::OK: + return "No error"; + case TransactionError::MISSING_INPUTS: + return "Missing inputs"; + case TransactionError::ALREADY_IN_CHAIN: + return "Transaction already in block chain"; + case TransactionError::P2P_DISABLED: + return "Peer-to-peer functionality missing or disabled"; + case TransactionError::MEMPOOL_REJECTED: + return "Transaction rejected by AcceptToMemoryPool"; + case TransactionError::MEMPOOL_ERROR: + return "AcceptToMemoryPool failed"; + case TransactionError::INVALID_PSBT: + return "PSBT is not sane"; + case TransactionError::PSBT_MISMATCH: + return "PSBTs not compatible (different transactions)"; + case TransactionError::SIGHASH_MISMATCH: + return "Specified sighash value does not match existing value"; + // no default case, so the compiler can warn about missing cases + } + assert(false); +} + +TransactionError BroadcastTransaction(const CTransactionRef tx, uint256& hashTx, std::string& err_string, const CAmount& highfee) +{ + std::promise<void> promise; + hashTx = tx->GetHash(); + + { // cs_main scope + LOCK(cs_main); + CCoinsViewCache &view = *pcoinsTip; + bool fHaveChain = false; + for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { + const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); + fHaveChain = !existingCoin.IsSpent(); + } + bool fHaveMempool = mempool.exists(hashTx); + if (!fHaveMempool && !fHaveChain) { + // push to local node and sync with wallets + CValidationState state; + bool fMissingInputs; + if (!AcceptToMemoryPool(mempool, state, std::move(tx), &fMissingInputs, + nullptr /* plTxnReplaced */, false /* bypass_limits */, highfee)) { + if (state.IsInvalid()) { + err_string = FormatStateMessage(state); + return TransactionError::MEMPOOL_REJECTED; + } else { + if (fMissingInputs) { + return TransactionError::MISSING_INPUTS; + } + err_string = FormatStateMessage(state); + return TransactionError::MEMPOOL_ERROR; + } + } else { + // If wallet is enabled, ensure that the wallet has been made aware + // of the new transaction prior to returning. This prevents a race + // where a user might call sendrawtransaction with a transaction + // to/from their wallet, immediately call some wallet RPC, and get + // a stale result because callbacks have not yet been processed. + CallFunctionInValidationInterfaceQueue([&promise] { + promise.set_value(); + }); + } + } else if (fHaveChain) { + return TransactionError::ALREADY_IN_CHAIN; + } else { + // Make sure we don't block forever if re-sending + // a transaction already in mempool. + promise.set_value(); + } + + } // cs_main + + promise.get_future().wait(); + + if (!g_connman) { + return TransactionError::P2P_DISABLED; + } + + CInv inv(MSG_TX, hashTx); + g_connman->ForEachNode([&inv](CNode* pnode) { + pnode->PushInventory(inv); + }); + + return TransactionError::OK; +} diff --git a/src/node/transaction.h b/src/node/transaction.h new file mode 100644 index 0000000000..3457ececa4 --- /dev/null +++ b/src/node/transaction.h @@ -0,0 +1,37 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_TRANSACTION_H +#define BITCOIN_NODE_TRANSACTION_H + +#include <attributes.h> +#include <primitives/transaction.h> +#include <uint256.h> + +enum class TransactionError { + OK, //!< No error + MISSING_INPUTS, + ALREADY_IN_CHAIN, + P2P_DISABLED, + MEMPOOL_REJECTED, + MEMPOOL_ERROR, + INVALID_PSBT, + PSBT_MISMATCH, + SIGHASH_MISMATCH, +}; + +std::string TransactionErrorString(const TransactionError error); + +/** + * Broadcast a transaction + * + * @param[in] tx the transaction to broadcast + * @param[out] &txid the txid of the transaction, if successfully broadcast + * @param[out] &err_string reference to std::string to fill with error string if available + * @param[in] highfee Reject txs with fees higher than this (if 0, accept any fee) + * return error + */ +NODISCARD TransactionError BroadcastTransaction(CTransactionRef tx, uint256& txid, std::string& err_string, const CAmount& highfee); + +#endif // BITCOIN_NODE_TRANSACTION_H diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 0dc130d104..c73a97fd7d 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -14,7 +14,7 @@ bool SignalsOptInRBF(const CTransaction &tx) return false; } -RBFTransactionState IsRBFOptIn(const CTransaction &tx, CTxMemPool &pool) +RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) { AssertLockHeld(pool.cs); diff --git a/src/policy/rbf.h b/src/policy/rbf.h index 581f489e12..a4f8777310 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -23,6 +23,6 @@ bool SignalsOptInRBF(const CTransaction &tx); // according to BIP 125 // This involves checking sequence numbers of the transaction, as well // as the sequence numbers of all in-mempool ancestors. -RBFTransactionState IsRBFOptIn(const CTransaction &tx, CTxMemPool &pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); +RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); #endif // BITCOIN_POLICY_RBF_H diff --git a/src/psbt.cpp b/src/psbt.cpp new file mode 100644 index 0000000000..0fb7d49d7d --- /dev/null +++ b/src/psbt.cpp @@ -0,0 +1,327 @@ +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <psbt.h> +#include <util/strencodings.h> + +PartiallySignedTransaction::PartiallySignedTransaction(const CMutableTransaction& tx) : tx(tx) +{ + inputs.resize(tx.vin.size()); + outputs.resize(tx.vout.size()); +} + +bool PartiallySignedTransaction::IsNull() const +{ + return !tx && inputs.empty() && outputs.empty() && unknown.empty(); +} + +bool PartiallySignedTransaction::Merge(const PartiallySignedTransaction& psbt) +{ + // Prohibited to merge two PSBTs over different transactions + if (tx->GetHash() != psbt.tx->GetHash()) { + return false; + } + + for (unsigned int i = 0; i < inputs.size(); ++i) { + inputs[i].Merge(psbt.inputs[i]); + } + for (unsigned int i = 0; i < outputs.size(); ++i) { + outputs[i].Merge(psbt.outputs[i]); + } + unknown.insert(psbt.unknown.begin(), psbt.unknown.end()); + + return true; +} + +bool PartiallySignedTransaction::IsSane() const +{ + for (PSBTInput input : inputs) { + if (!input.IsSane()) return false; + } + return true; +} + +bool PartiallySignedTransaction::AddInput(const CTxIn& txin, PSBTInput& psbtin) +{ + if (std::find(tx->vin.begin(), tx->vin.end(), txin) != tx->vin.end()) { + return false; + } + tx->vin.push_back(txin); + psbtin.partial_sigs.clear(); + psbtin.final_script_sig.clear(); + psbtin.final_script_witness.SetNull(); + inputs.push_back(psbtin); + return true; +} + +bool PartiallySignedTransaction::AddOutput(const CTxOut& txout, const PSBTOutput& psbtout) +{ + tx->vout.push_back(txout); + outputs.push_back(psbtout); + return true; +} + +bool PartiallySignedTransaction::GetInputUTXO(CTxOut& utxo, int input_index) const +{ + PSBTInput input = inputs[input_index]; + int prevout_index = tx->vin[input_index].prevout.n; + if (input.non_witness_utxo) { + utxo = input.non_witness_utxo->vout[prevout_index]; + } else if (!input.witness_utxo.IsNull()) { + utxo = input.witness_utxo; + } else { + return false; + } + return true; +} + +bool PSBTInput::IsNull() const +{ + return !non_witness_utxo && witness_utxo.IsNull() && partial_sigs.empty() && unknown.empty() && hd_keypaths.empty() && redeem_script.empty() && witness_script.empty(); +} + +void PSBTInput::FillSignatureData(SignatureData& sigdata) const +{ + if (!final_script_sig.empty()) { + sigdata.scriptSig = final_script_sig; + sigdata.complete = true; + } + if (!final_script_witness.IsNull()) { + sigdata.scriptWitness = final_script_witness; + sigdata.complete = true; + } + if (sigdata.complete) { + return; + } + + sigdata.signatures.insert(partial_sigs.begin(), partial_sigs.end()); + if (!redeem_script.empty()) { + sigdata.redeem_script = redeem_script; + } + if (!witness_script.empty()) { + sigdata.witness_script = witness_script; + } + for (const auto& key_pair : hd_keypaths) { + sigdata.misc_pubkeys.emplace(key_pair.first.GetID(), key_pair); + } +} + +void PSBTInput::FromSignatureData(const SignatureData& sigdata) +{ + if (sigdata.complete) { + partial_sigs.clear(); + hd_keypaths.clear(); + redeem_script.clear(); + witness_script.clear(); + + if (!sigdata.scriptSig.empty()) { + final_script_sig = sigdata.scriptSig; + } + if (!sigdata.scriptWitness.IsNull()) { + final_script_witness = sigdata.scriptWitness; + } + return; + } + + partial_sigs.insert(sigdata.signatures.begin(), sigdata.signatures.end()); + if (redeem_script.empty() && !sigdata.redeem_script.empty()) { + redeem_script = sigdata.redeem_script; + } + if (witness_script.empty() && !sigdata.witness_script.empty()) { + witness_script = sigdata.witness_script; + } + for (const auto& entry : sigdata.misc_pubkeys) { + hd_keypaths.emplace(entry.second); + } +} + +void PSBTInput::Merge(const PSBTInput& input) +{ + if (!non_witness_utxo && input.non_witness_utxo) non_witness_utxo = input.non_witness_utxo; + if (witness_utxo.IsNull() && !input.witness_utxo.IsNull()) { + witness_utxo = input.witness_utxo; + non_witness_utxo = nullptr; // Clear out any non-witness utxo when we set a witness one. + } + + partial_sigs.insert(input.partial_sigs.begin(), input.partial_sigs.end()); + hd_keypaths.insert(input.hd_keypaths.begin(), input.hd_keypaths.end()); + unknown.insert(input.unknown.begin(), input.unknown.end()); + + if (redeem_script.empty() && !input.redeem_script.empty()) redeem_script = input.redeem_script; + if (witness_script.empty() && !input.witness_script.empty()) witness_script = input.witness_script; + if (final_script_sig.empty() && !input.final_script_sig.empty()) final_script_sig = input.final_script_sig; + if (final_script_witness.IsNull() && !input.final_script_witness.IsNull()) final_script_witness = input.final_script_witness; +} + +bool PSBTInput::IsSane() const +{ + // Cannot have both witness and non-witness utxos + if (!witness_utxo.IsNull() && non_witness_utxo) return false; + + // If we have a witness_script or a scriptWitness, we must also have a witness utxo + if (!witness_script.empty() && witness_utxo.IsNull()) return false; + if (!final_script_witness.IsNull() && witness_utxo.IsNull()) return false; + + return true; +} + +void PSBTOutput::FillSignatureData(SignatureData& sigdata) const +{ + if (!redeem_script.empty()) { + sigdata.redeem_script = redeem_script; + } + if (!witness_script.empty()) { + sigdata.witness_script = witness_script; + } + for (const auto& key_pair : hd_keypaths) { + sigdata.misc_pubkeys.emplace(key_pair.first.GetID(), key_pair); + } +} + +void PSBTOutput::FromSignatureData(const SignatureData& sigdata) +{ + if (redeem_script.empty() && !sigdata.redeem_script.empty()) { + redeem_script = sigdata.redeem_script; + } + if (witness_script.empty() && !sigdata.witness_script.empty()) { + witness_script = sigdata.witness_script; + } + for (const auto& entry : sigdata.misc_pubkeys) { + hd_keypaths.emplace(entry.second); + } +} + +bool PSBTOutput::IsNull() const +{ + return redeem_script.empty() && witness_script.empty() && hd_keypaths.empty() && unknown.empty(); +} + +void PSBTOutput::Merge(const PSBTOutput& output) +{ + hd_keypaths.insert(output.hd_keypaths.begin(), output.hd_keypaths.end()); + unknown.insert(output.unknown.begin(), output.unknown.end()); + + if (redeem_script.empty() && !output.redeem_script.empty()) redeem_script = output.redeem_script; + if (witness_script.empty() && !output.witness_script.empty()) witness_script = output.witness_script; +} +bool PSBTInputSigned(PSBTInput& input) +{ + return !input.final_script_sig.empty() || !input.final_script_witness.IsNull(); +} + +bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash, SignatureData* out_sigdata, bool use_dummy) +{ + PSBTInput& input = psbt.inputs.at(index); + const CMutableTransaction& tx = *psbt.tx; + + if (PSBTInputSigned(input)) { + return true; + } + + // Fill SignatureData with input info + SignatureData sigdata; + input.FillSignatureData(sigdata); + + // Get UTXO + bool require_witness_sig = false; + CTxOut utxo; + + // Verify input sanity, which checks that at most one of witness or non-witness utxos is provided. + if (!input.IsSane()) { + return false; + } + + if (input.non_witness_utxo) { + // If we're taking our information from a non-witness UTXO, verify that it matches the prevout. + COutPoint prevout = tx.vin[index].prevout; + if (input.non_witness_utxo->GetHash() != prevout.hash) { + return false; + } + utxo = input.non_witness_utxo->vout[prevout.n]; + } else if (!input.witness_utxo.IsNull()) { + utxo = input.witness_utxo; + // When we're taking our information from a witness UTXO, we can't verify it is actually data from + // the output being spent. This is safe in case a witness signature is produced (which includes this + // information directly in the hash), but not for non-witness signatures. Remember that we require + // a witness signature in this situation. + require_witness_sig = true; + } else { + return false; + } + + sigdata.witness = false; + bool sig_complete; + if (use_dummy) { + sig_complete = ProduceSignature(provider, DUMMY_SIGNATURE_CREATOR, utxo.scriptPubKey, sigdata); + } else { + MutableTransactionSignatureCreator creator(&tx, index, utxo.nValue, sighash); + sig_complete = ProduceSignature(provider, creator, utxo.scriptPubKey, sigdata); + } + // Verify that a witness signature was produced in case one was required. + if (require_witness_sig && !sigdata.witness) return false; + input.FromSignatureData(sigdata); + + // If we have a witness signature, use the smaller witness UTXO. + if (sigdata.witness) { + input.witness_utxo = utxo; + input.non_witness_utxo = nullptr; + } + + // Fill in the missing info + if (out_sigdata) { + out_sigdata->missing_pubkeys = sigdata.missing_pubkeys; + out_sigdata->missing_sigs = sigdata.missing_sigs; + out_sigdata->missing_redeem_script = sigdata.missing_redeem_script; + out_sigdata->missing_witness_script = sigdata.missing_witness_script; + } + + return sig_complete; +} + +bool FinalizePSBT(PartiallySignedTransaction& psbtx) +{ + // Finalize input signatures -- in case we have partial signatures that add up to a complete + // signature, but have not combined them yet (e.g. because the combiner that created this + // PartiallySignedTransaction did not understand them), this will combine them into a final + // script. + bool complete = true; + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + complete &= SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, SIGHASH_ALL); + } + + return complete; +} + +bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransaction& result) +{ + // It's not safe to extract a PSBT that isn't finalized, and there's no easy way to check + // whether a PSBT is finalized without finalizing it, so we just do this. + if (!FinalizePSBT(psbtx)) { + return false; + } + + result = *psbtx.tx; + for (unsigned int i = 0; i < result.vin.size(); ++i) { + result.vin[i].scriptSig = psbtx.inputs[i].final_script_sig; + result.vin[i].scriptWitness = psbtx.inputs[i].final_script_witness; + } + return true; +} + +TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs) +{ + out = psbtxs[0]; // Copy the first one + + // Merge + for (auto it = std::next(psbtxs.begin()); it != psbtxs.end(); ++it) { + if (!out.Merge(*it)) { + return TransactionError::PSBT_MISMATCH; + } + } + if (!out.IsSane()) { + return TransactionError::INVALID_PSBT; + } + + return TransactionError::OK; +} diff --git a/src/psbt.h b/src/psbt.h new file mode 100644 index 0000000000..c889dad361 --- /dev/null +++ b/src/psbt.h @@ -0,0 +1,583 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_PSBT_H +#define BITCOIN_PSBT_H + +#include <attributes.h> +#include <node/transaction.h> +#include <primitives/transaction.h> +#include <pubkey.h> +#include <script/sign.h> + +// Magic bytes +static constexpr uint8_t PSBT_MAGIC_BYTES[5] = {'p', 's', 'b', 't', 0xff}; + +// Global types +static constexpr uint8_t PSBT_GLOBAL_UNSIGNED_TX = 0x00; + +// Input types +static constexpr uint8_t PSBT_IN_NON_WITNESS_UTXO = 0x00; +static constexpr uint8_t PSBT_IN_WITNESS_UTXO = 0x01; +static constexpr uint8_t PSBT_IN_PARTIAL_SIG = 0x02; +static constexpr uint8_t PSBT_IN_SIGHASH = 0x03; +static constexpr uint8_t PSBT_IN_REDEEMSCRIPT = 0x04; +static constexpr uint8_t PSBT_IN_WITNESSSCRIPT = 0x05; +static constexpr uint8_t PSBT_IN_BIP32_DERIVATION = 0x06; +static constexpr uint8_t PSBT_IN_SCRIPTSIG = 0x07; +static constexpr uint8_t PSBT_IN_SCRIPTWITNESS = 0x08; + +// Output types +static constexpr uint8_t PSBT_OUT_REDEEMSCRIPT = 0x00; +static constexpr uint8_t PSBT_OUT_WITNESSSCRIPT = 0x01; +static constexpr uint8_t PSBT_OUT_BIP32_DERIVATION = 0x02; + +// The separator is 0x00. Reading this in means that the unserializer can interpret it +// as a 0 length key which indicates that this is the separator. The separator has no value. +static constexpr uint8_t PSBT_SEPARATOR = 0x00; + +/** A structure for PSBTs which contain per-input information */ +struct PSBTInput +{ + CTransactionRef non_witness_utxo; + CTxOut witness_utxo; + CScript redeem_script; + CScript witness_script; + CScript final_script_sig; + CScriptWitness final_script_witness; + std::map<CPubKey, KeyOriginInfo> hd_keypaths; + std::map<CKeyID, SigPair> partial_sigs; + std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; + int sighash_type = 0; + + bool IsNull() const; + void FillSignatureData(SignatureData& sigdata) const; + void FromSignatureData(const SignatureData& sigdata); + void Merge(const PSBTInput& input); + bool IsSane() const; + PSBTInput() {} + + template <typename Stream> + inline void Serialize(Stream& s) const { + // Write the utxo + // If there is a non-witness utxo, then don't add the witness one. + if (non_witness_utxo) { + SerializeToVector(s, PSBT_IN_NON_WITNESS_UTXO); + OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); + SerializeToVector(os, non_witness_utxo); + } else if (!witness_utxo.IsNull()) { + SerializeToVector(s, PSBT_IN_WITNESS_UTXO); + SerializeToVector(s, witness_utxo); + } + + if (final_script_sig.empty() && final_script_witness.IsNull()) { + // Write any partial signatures + for (auto sig_pair : partial_sigs) { + SerializeToVector(s, PSBT_IN_PARTIAL_SIG, MakeSpan(sig_pair.second.first)); + s << sig_pair.second.second; + } + + // Write the sighash type + if (sighash_type > 0) { + SerializeToVector(s, PSBT_IN_SIGHASH); + SerializeToVector(s, sighash_type); + } + + // Write the redeem script + if (!redeem_script.empty()) { + SerializeToVector(s, PSBT_IN_REDEEMSCRIPT); + s << redeem_script; + } + + // Write the witness script + if (!witness_script.empty()) { + SerializeToVector(s, PSBT_IN_WITNESSSCRIPT); + s << witness_script; + } + + // Write any hd keypaths + SerializeHDKeypaths(s, hd_keypaths, PSBT_IN_BIP32_DERIVATION); + } + + // Write script sig + if (!final_script_sig.empty()) { + SerializeToVector(s, PSBT_IN_SCRIPTSIG); + s << final_script_sig; + } + // write script witness + if (!final_script_witness.IsNull()) { + SerializeToVector(s, PSBT_IN_SCRIPTWITNESS); + SerializeToVector(s, final_script_witness.stack); + } + + // Write unknown things + for (auto& entry : unknown) { + s << entry.first; + s << entry.second; + } + + s << PSBT_SEPARATOR; + } + + + template <typename Stream> + inline void Unserialize(Stream& s) { + // Read loop + bool found_sep = false; + while(!s.empty()) { + // Read + std::vector<unsigned char> key; + s >> key; + + // the key is empty if that was actually a separator byte + // This is a special case for key lengths 0 as those are not allowed (except for separator) + if (key.empty()) { + found_sep = true; + break; + } + + // First byte of key is the type + unsigned char type = key[0]; + + // Do stuff based on type + switch(type) { + case PSBT_IN_NON_WITNESS_UTXO: + { + if (non_witness_utxo) { + throw std::ios_base::failure("Duplicate Key, input non-witness utxo already provided"); + } else if (key.size() != 1) { + throw std::ios_base::failure("Non-witness utxo key is more than one byte type"); + } + // Set the stream to unserialize with witness since this is always a valid network transaction + OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() & ~SERIALIZE_TRANSACTION_NO_WITNESS); + UnserializeFromVector(os, non_witness_utxo); + break; + } + case PSBT_IN_WITNESS_UTXO: + if (!witness_utxo.IsNull()) { + throw std::ios_base::failure("Duplicate Key, input witness utxo already provided"); + } else if (key.size() != 1) { + throw std::ios_base::failure("Witness utxo key is more than one byte type"); + } + UnserializeFromVector(s, witness_utxo); + break; + case PSBT_IN_PARTIAL_SIG: + { + // Make sure that the key is the size of pubkey + 1 + if (key.size() != CPubKey::PUBLIC_KEY_SIZE + 1 && key.size() != CPubKey::COMPRESSED_PUBLIC_KEY_SIZE + 1) { + throw std::ios_base::failure("Size of key was not the expected size for the type partial signature pubkey"); + } + // Read in the pubkey from key + CPubKey pubkey(key.begin() + 1, key.end()); + if (!pubkey.IsFullyValid()) { + throw std::ios_base::failure("Invalid pubkey"); + } + if (partial_sigs.count(pubkey.GetID()) > 0) { + throw std::ios_base::failure("Duplicate Key, input partial signature for pubkey already provided"); + } + + // Read in the signature from value + std::vector<unsigned char> sig; + s >> sig; + + // Add to list + partial_sigs.emplace(pubkey.GetID(), SigPair(pubkey, std::move(sig))); + break; + } + case PSBT_IN_SIGHASH: + if (sighash_type > 0) { + throw std::ios_base::failure("Duplicate Key, input sighash type already provided"); + } else if (key.size() != 1) { + throw std::ios_base::failure("Sighash type key is more than one byte type"); + } + UnserializeFromVector(s, sighash_type); + break; + case PSBT_IN_REDEEMSCRIPT: + { + if (!redeem_script.empty()) { + throw std::ios_base::failure("Duplicate Key, input redeemScript already provided"); + } else if (key.size() != 1) { + throw std::ios_base::failure("Input redeemScript key is more than one byte type"); + } + s >> redeem_script; + break; + } + case PSBT_IN_WITNESSSCRIPT: + { + if (!witness_script.empty()) { + throw std::ios_base::failure("Duplicate Key, input witnessScript already provided"); + } else if (key.size() != 1) { + throw std::ios_base::failure("Input witnessScript key is more than one byte type"); + } + s >> witness_script; + break; + } + case PSBT_IN_BIP32_DERIVATION: + { + DeserializeHDKeypaths(s, key, hd_keypaths); + break; + } + case PSBT_IN_SCRIPTSIG: + { + if (!final_script_sig.empty()) { + throw std::ios_base::failure("Duplicate Key, input final scriptSig already provided"); + } else if (key.size() != 1) { + throw std::ios_base::failure("Final scriptSig key is more than one byte type"); + } + s >> final_script_sig; + break; + } + case PSBT_IN_SCRIPTWITNESS: + { + if (!final_script_witness.IsNull()) { + throw std::ios_base::failure("Duplicate Key, input final scriptWitness already provided"); + } else if (key.size() != 1) { + throw std::ios_base::failure("Final scriptWitness key is more than one byte type"); + } + UnserializeFromVector(s, final_script_witness.stack); + break; + } + // Unknown stuff + default: + if (unknown.count(key) > 0) { + throw std::ios_base::failure("Duplicate Key, key for unknown value already provided"); + } + // Read in the value + std::vector<unsigned char> val_bytes; + s >> val_bytes; + unknown.emplace(std::move(key), std::move(val_bytes)); + break; + } + } + + if (!found_sep) { + throw std::ios_base::failure("Separator is missing at the end of an input map"); + } + } + + template <typename Stream> + PSBTInput(deserialize_type, Stream& s) { + Unserialize(s); + } +}; + +/** A structure for PSBTs which contains per output information */ +struct PSBTOutput +{ + CScript redeem_script; + CScript witness_script; + std::map<CPubKey, KeyOriginInfo> hd_keypaths; + std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; + + bool IsNull() const; + void FillSignatureData(SignatureData& sigdata) const; + void FromSignatureData(const SignatureData& sigdata); + void Merge(const PSBTOutput& output); + bool IsSane() const; + PSBTOutput() {} + + template <typename Stream> + inline void Serialize(Stream& s) const { + // Write the redeem script + if (!redeem_script.empty()) { + SerializeToVector(s, PSBT_OUT_REDEEMSCRIPT); + s << redeem_script; + } + + // Write the witness script + if (!witness_script.empty()) { + SerializeToVector(s, PSBT_OUT_WITNESSSCRIPT); + s << witness_script; + } + + // Write any hd keypaths + SerializeHDKeypaths(s, hd_keypaths, PSBT_OUT_BIP32_DERIVATION); + + // Write unknown things + for (auto& entry : unknown) { + s << entry.first; + s << entry.second; + } + + s << PSBT_SEPARATOR; + } + + + template <typename Stream> + inline void Unserialize(Stream& s) { + // Read loop + bool found_sep = false; + while(!s.empty()) { + // Read + std::vector<unsigned char> key; + s >> key; + + // the key is empty if that was actually a separator byte + // This is a special case for key lengths 0 as those are not allowed (except for separator) + if (key.empty()) { + found_sep = true; + break; + } + + // First byte of key is the type + unsigned char type = key[0]; + + // Do stuff based on type + switch(type) { + case PSBT_OUT_REDEEMSCRIPT: + { + if (!redeem_script.empty()) { + throw std::ios_base::failure("Duplicate Key, output redeemScript already provided"); + } else if (key.size() != 1) { + throw std::ios_base::failure("Output redeemScript key is more than one byte type"); + } + s >> redeem_script; + break; + } + case PSBT_OUT_WITNESSSCRIPT: + { + if (!witness_script.empty()) { + throw std::ios_base::failure("Duplicate Key, output witnessScript already provided"); + } else if (key.size() != 1) { + throw std::ios_base::failure("Output witnessScript key is more than one byte type"); + } + s >> witness_script; + break; + } + case PSBT_OUT_BIP32_DERIVATION: + { + DeserializeHDKeypaths(s, key, hd_keypaths); + break; + } + // Unknown stuff + default: { + if (unknown.count(key) > 0) { + throw std::ios_base::failure("Duplicate Key, key for unknown value already provided"); + } + // Read in the value + std::vector<unsigned char> val_bytes; + s >> val_bytes; + unknown.emplace(std::move(key), std::move(val_bytes)); + break; + } + } + } + + if (!found_sep) { + throw std::ios_base::failure("Separator is missing at the end of an output map"); + } + } + + template <typename Stream> + PSBTOutput(deserialize_type, Stream& s) { + Unserialize(s); + } +}; + +/** A version of CTransaction with the PSBT format*/ +struct PartiallySignedTransaction +{ + boost::optional<CMutableTransaction> tx; + std::vector<PSBTInput> inputs; + std::vector<PSBTOutput> outputs; + std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; + + bool IsNull() const; + + /** Merge psbt into this. The two psbts must have the same underlying CTransaction (i.e. the + * same actual Bitcoin transaction.) Returns true if the merge succeeded, false otherwise. */ + NODISCARD bool Merge(const PartiallySignedTransaction& psbt); + bool IsSane() const; + bool AddInput(const CTxIn& txin, PSBTInput& psbtin); + bool AddOutput(const CTxOut& txout, const PSBTOutput& psbtout); + PartiallySignedTransaction() {} + PartiallySignedTransaction(const PartiallySignedTransaction& psbt_in) : tx(psbt_in.tx), inputs(psbt_in.inputs), outputs(psbt_in.outputs), unknown(psbt_in.unknown) {} + explicit PartiallySignedTransaction(const CMutableTransaction& tx); + /** + * Finds the UTXO for a given input index + * + * @param[out] utxo The UTXO of the input if found + * @param[in] input_index Index of the input to retrieve the UTXO of + * @return Whether the UTXO for the specified input was found + */ + bool GetInputUTXO(CTxOut& utxo, int input_index) const; + + template <typename Stream> + inline void Serialize(Stream& s) const { + + // magic bytes + s << PSBT_MAGIC_BYTES; + + // unsigned tx flag + SerializeToVector(s, PSBT_GLOBAL_UNSIGNED_TX); + + // Write serialized tx to a stream + OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); + SerializeToVector(os, *tx); + + // Write the unknown things + for (auto& entry : unknown) { + s << entry.first; + s << entry.second; + } + + // Separator + s << PSBT_SEPARATOR; + + // Write inputs + for (const PSBTInput& input : inputs) { + s << input; + } + // Write outputs + for (const PSBTOutput& output : outputs) { + s << output; + } + } + + + template <typename Stream> + inline void Unserialize(Stream& s) { + // Read the magic bytes + uint8_t magic[5]; + s >> magic; + if (!std::equal(magic, magic + 5, PSBT_MAGIC_BYTES)) { + throw std::ios_base::failure("Invalid PSBT magic bytes"); + } + + // Read global data + bool found_sep = false; + while(!s.empty()) { + // Read + std::vector<unsigned char> key; + s >> key; + + // the key is empty if that was actually a separator byte + // This is a special case for key lengths 0 as those are not allowed (except for separator) + if (key.empty()) { + found_sep = true; + break; + } + + // First byte of key is the type + unsigned char type = key[0]; + + // Do stuff based on type + switch(type) { + case PSBT_GLOBAL_UNSIGNED_TX: + { + if (tx) { + throw std::ios_base::failure("Duplicate Key, unsigned tx already provided"); + } else if (key.size() != 1) { + throw std::ios_base::failure("Global unsigned tx key is more than one byte type"); + } + CMutableTransaction mtx; + // Set the stream to serialize with non-witness since this should always be non-witness + OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); + UnserializeFromVector(os, mtx); + tx = std::move(mtx); + // Make sure that all scriptSigs and scriptWitnesses are empty + for (const CTxIn& txin : tx->vin) { + if (!txin.scriptSig.empty() || !txin.scriptWitness.IsNull()) { + throw std::ios_base::failure("Unsigned tx does not have empty scriptSigs and scriptWitnesses."); + } + } + break; + } + // Unknown stuff + default: { + if (unknown.count(key) > 0) { + throw std::ios_base::failure("Duplicate Key, key for unknown value already provided"); + } + // Read in the value + std::vector<unsigned char> val_bytes; + s >> val_bytes; + unknown.emplace(std::move(key), std::move(val_bytes)); + } + } + } + + if (!found_sep) { + throw std::ios_base::failure("Separator is missing at the end of the global map"); + } + + // Make sure that we got an unsigned tx + if (!tx) { + throw std::ios_base::failure("No unsigned transcation was provided"); + } + + // Read input data + unsigned int i = 0; + while (!s.empty() && i < tx->vin.size()) { + PSBTInput input; + s >> input; + inputs.push_back(input); + + // Make sure the non-witness utxo matches the outpoint + if (input.non_witness_utxo && input.non_witness_utxo->GetHash() != tx->vin[i].prevout.hash) { + throw std::ios_base::failure("Non-witness UTXO does not match outpoint hash"); + } + ++i; + } + // Make sure that the number of inputs matches the number of inputs in the transaction + if (inputs.size() != tx->vin.size()) { + throw std::ios_base::failure("Inputs provided does not match the number of inputs in transaction."); + } + + // Read output data + i = 0; + while (!s.empty() && i < tx->vout.size()) { + PSBTOutput output; + s >> output; + outputs.push_back(output); + ++i; + } + // Make sure that the number of outputs matches the number of outputs in the transaction + if (outputs.size() != tx->vout.size()) { + throw std::ios_base::failure("Outputs provided does not match the number of outputs in transaction."); + } + // Sanity check + if (!IsSane()) { + throw std::ios_base::failure("PSBT is not sane."); + } + } + + template <typename Stream> + PartiallySignedTransaction(deserialize_type, Stream& s) { + Unserialize(s); + } +}; + +/** Checks whether a PSBTInput is already signed. */ +bool PSBTInputSigned(PSBTInput& input); + +/** Signs a PSBTInput, verifying that all provided data matches what is being signed. */ +bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash = SIGHASH_ALL, SignatureData* out_sigdata = nullptr, bool use_dummy = false); + +/** + * Finalizes a PSBT if possible, combining partial signatures. + * + * @param[in,out] &psbtx reference to PartiallySignedTransaction to finalize + * return True if the PSBT is now complete, false otherwise + */ +bool FinalizePSBT(PartiallySignedTransaction& psbtx); + +/** + * Finalizes a PSBT if possible, and extracts it to a CMutableTransaction if it could be finalized. + * + * @param[in] &psbtx reference to PartiallySignedTransaction + * @param[out] result CMutableTransaction representing the complete transaction, if successful + * @return True if we successfully extracted the transaction, false otherwise + */ +bool FinalizeAndExtractPSBT(PartiallySignedTransaction& psbtx, CMutableTransaction& result); + +/** + * Combines PSBTs with the same underlying transaction, resulting in a single PSBT with all partial signatures from each input. + * + * @param[out] &out the combined PSBT, if successful + * @param[in] psbtxs the PSBTs to combine + * @return error (OK if we successfully combined the transactions, other error if they were not compatible) + */ +NODISCARD TransactionError CombinePSBTs(PartiallySignedTransaction& out, const std::vector<PartiallySignedTransaction>& psbtxs); + +#endif // BITCOIN_PSBT_H diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 85d79ee26c..1b063771ef 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -55,9 +55,6 @@ #if defined(QT_STATICPLUGIN) #include <QtPlugin> -#if QT_VERSION < 0x050400 -Q_IMPORT_PLUGIN(AccessibleFactory) -#endif #if defined(QT_QPA_PLATFORM_XCB) Q_IMPORT_PLUGIN(QXcbIntegrationPlugin); #elif defined(QT_QPA_PLATFORM_WINDOWS) @@ -456,7 +453,7 @@ int GuiMain(int argc, char* argv[]) // IMPORTANT if it is no longer a typedef use the normal variant above qRegisterMetaType< CAmount >("CAmount"); qRegisterMetaType< std::function<void()> >("std::function<void()>"); - + qRegisterMetaType<QMessageBox::Icon>("QMessageBox::Icon"); /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: node->setupServerArgs(); diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index ba7e8c7daf..abf9136eee 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -334,6 +334,13 @@ void BitcoinGUI::createActions() openAction = new QAction(platformStyle->TextColorIcon(":/icons/open"), tr("Open &URI..."), this); openAction->setStatusTip(tr("Open a bitcoin: URI or payment request")); + m_open_wallet_action = new QAction(tr("Open Wallet"), this); + m_open_wallet_action->setMenu(new QMenu(this)); + m_open_wallet_action->setStatusTip(tr("Open a wallet")); + + m_close_wallet_action = new QAction(tr("Close Wallet..."), this); + m_close_wallet_action->setStatusTip(tr("Close wallet")); + showHelpMessageAction = new QAction(platformStyle->TextColorIcon(":/icons/info"), tr("&Command-line options"), this); showHelpMessageAction->setMenuRole(QAction::NoRole); showHelpMessageAction->setStatusTip(tr("Show the %1 help message to get a list with possible Bitcoin command-line options").arg(tr(PACKAGE_NAME))); @@ -361,6 +368,42 @@ void BitcoinGUI::createActions() connect(usedSendingAddressesAction, &QAction::triggered, walletFrame, &WalletFrame::usedSendingAddresses); connect(usedReceivingAddressesAction, &QAction::triggered, walletFrame, &WalletFrame::usedReceivingAddresses); connect(openAction, &QAction::triggered, this, &BitcoinGUI::openClicked); + connect(m_open_wallet_action->menu(), &QMenu::aboutToShow, [this] { + m_open_wallet_action->menu()->clear(); + for (std::string path : m_wallet_controller->getWalletsAvailableToOpen()) { + QString name = path.empty() ? QString("["+tr("default wallet")+"]") : QString::fromStdString(path); + QAction* action = m_open_wallet_action->menu()->addAction(name); + connect(action, &QAction::triggered, [this, name, path] { + OpenWalletActivity* activity = m_wallet_controller->openWallet(path); + + QProgressDialog* dialog = new QProgressDialog(this); + dialog->setLabelText(tr("Opening Wallet <b>%1</b>...").arg(name.toHtmlEscaped())); + dialog->setRange(0, 0); + dialog->setCancelButton(nullptr); + dialog->setWindowModality(Qt::ApplicationModal); + dialog->show(); + + connect(activity, &OpenWalletActivity::message, this, [this] (QMessageBox::Icon icon, QString text) { + QMessageBox box; + box.setIcon(icon); + box.setText(tr("Open Wallet Failed")); + box.setInformativeText(text); + box.setStandardButtons(QMessageBox::Ok); + box.setDefaultButton(QMessageBox::Ok); + connect(this, &QObject::destroyed, &box, &QDialog::accept); + box.exec(); + }); + connect(activity, &OpenWalletActivity::opened, this, &BitcoinGUI::setCurrentWallet); + connect(activity, &OpenWalletActivity::finished, activity, &QObject::deleteLater); + connect(activity, &OpenWalletActivity::finished, dialog, &QObject::deleteLater); + bool invoked = QMetaObject::invokeMethod(activity, "open"); + assert(invoked); + }); + } + }); + connect(m_close_wallet_action, &QAction::triggered, [this] { + m_wallet_controller->closeWallet(walletFrame->currentWalletModel(), this); + }); } #endif // ENABLE_WALLET @@ -382,6 +425,9 @@ void BitcoinGUI::createMenuBar() QMenu *file = appMenuBar->addMenu(tr("&File")); if(walletFrame) { + file->addAction(m_open_wallet_action); + file->addAction(m_close_wallet_action); + file->addSeparator(); file->addAction(openAction); file->addAction(backupWalletAction); file->addAction(signMessageAction); @@ -656,6 +702,7 @@ void BitcoinGUI::setWalletActionsEnabled(bool enabled) usedSendingAddressesAction->setEnabled(enabled); usedReceivingAddressesAction->setEnabled(enabled); openAction->setEnabled(enabled); + m_close_wallet_action->setEnabled(enabled); } void BitcoinGUI::createTrayIcon() diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index c31cefe603..b58ccbb455 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -147,6 +147,8 @@ private: QAction* openRPCConsoleAction = nullptr; QAction* openAction = nullptr; QAction* showHelpMessageAction = nullptr; + QAction* m_open_wallet_action{nullptr}; + QAction* m_close_wallet_action{nullptr}; QAction* m_wallet_selector_label_action = nullptr; QAction* m_wallet_selector_action = nullptr; diff --git a/src/qt/paymentserver.cpp b/src/qt/paymentserver.cpp index b1ec90ad36..43dccec4ea 100644 --- a/src/qt/paymentserver.cpp +++ b/src/qt/paymentserver.cpp @@ -292,9 +292,9 @@ void PaymentServer::handleURIOrFile(const QString& s) else if (s.startsWith(BITCOIN_IPC_PREFIX, Qt::CaseInsensitive)) // bitcoin: URI { QUrlQuery uri((QUrl(s))); +#ifdef ENABLE_BIP70 if (uri.hasQueryItem("r")) // payment request URI { -#ifdef ENABLE_BIP70 Q_EMIT message(tr("URI handling"), tr("You are using a BIP70 URL which will be unsupported in the future."), CClientUIInterface::ICON_WARNING); @@ -315,19 +315,23 @@ void PaymentServer::handleURIOrFile(const QString& s) tr("Payment request fetch URL is invalid: %1").arg(fetchUrl.toString()), CClientUIInterface::ICON_WARNING); } -#else - Q_EMIT message(tr("URI handling"), - tr("Cannot process payment request because BIP70 support was not compiled in."), - CClientUIInterface::ICON_WARNING); -#endif return; } - else // normal URI + else +#endif + // normal URI { SendCoinsRecipient recipient; if (GUIUtil::parseBitcoinURI(s, &recipient)) { if (!IsValidDestinationString(recipient.address.toStdString())) { +#ifndef ENABLE_BIP70 + if (uri.hasQueryItem("r")) { // payment request + Q_EMIT message(tr("URI handling"), + tr("Cannot process payment request because BIP70 support was not compiled in."), + CClientUIInterface::ICON_WARNING); + } +#endif Q_EMIT message(tr("URI handling"), tr("Invalid payment address %1").arg(recipient.address), CClientUIInterface::MSG_ERROR); } @@ -343,9 +347,9 @@ void PaymentServer::handleURIOrFile(const QString& s) } } -#ifdef ENABLE_BIP70 if (QFile::exists(s)) // payment request file { +#ifdef ENABLE_BIP70 PaymentRequestPlus request; SendCoinsRecipient recipient; if (!readPaymentRequestFromFile(s, request)) @@ -358,8 +362,12 @@ void PaymentServer::handleURIOrFile(const QString& s) Q_EMIT receivedPaymentRequest(recipient); return; - } +#else + Q_EMIT message(tr("Payment request file handling"), + tr("Cannot process payment request because BIP70 support was not compiled in."), + CClientUIInterface::ICON_WARNING); #endif + } } void PaymentServer::handleURIConnection() diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 3e414df1f0..7f5e92ea9f 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -95,6 +95,7 @@ void TestAddAddressesToSendBook() } auto check_addbook_size = [&wallet](int expected_size) { + LOCK(wallet->cs_wallet); QCOMPARE(static_cast<int>(wallet->mapAddressBook.size()), expected_size); }; diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 2c477a2e98..da25d83175 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -27,9 +27,7 @@ #include <QTest> #include <QTextEdit> #include <QtGlobal> -#if QT_VERSION >= 0x050000 #include <QtTest/QtTestWidgets> -#endif #include <QtTest/QtTestGui> #include <new> #include <string> diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 173c814f1e..f8a9c25303 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -120,7 +120,6 @@ void RPCNestedTests::rpcNestedTests() RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest( abc , cba )"); QVERIFY(result == "[\"abc\",\"cba\"]"); -#if QT_VERSION >= 0x050300 // do the QVERIFY_EXCEPTION_THROWN checks only with Qt5.3 and higher (QVERIFY_EXCEPTION_THROWN was introduced in Qt5.3) QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo() .\n"), std::runtime_error); //invalid syntax QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "getblockchaininfo() getblockchaininfo()"), std::runtime_error); //invalid syntax @@ -131,5 +130,4 @@ void RPCNestedTests::rpcNestedTests() QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest abc,,abc"), std::runtime_error); //don't tollerate empty arguments when using , QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest(abc,,abc)"), std::runtime_error); //don't tollerate empty arguments when using , QVERIFY_EXCEPTION_THROWN(RPCConsole::RPCExecuteCommandLine(*node, result, "rpcNestedTest(abc,,)"), std::runtime_error); //don't tollerate empty arguments when using , -#endif } diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index df2b7a3f9b..fab86a7912 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -9,6 +9,7 @@ #include <algorithm> +#include <QMessageBox> #include <QMutexLocker> #include <QThread> @@ -25,11 +26,17 @@ WalletController::WalletController(interfaces::Node& node, const PlatformStyle* for (std::unique_ptr<interfaces::Wallet>& wallet : m_node.getWallets()) { getOrCreateWallet(std::move(wallet)); } + + m_activity_thread.start(); } // Not using the default destructor because not all member types definitions are // available in the header, just forward declared. -WalletController::~WalletController() {} +WalletController::~WalletController() +{ + m_activity_thread.quit(); + m_activity_thread.wait(); +} std::vector<WalletModel*> WalletController::getWallets() const { @@ -37,6 +44,40 @@ std::vector<WalletModel*> WalletController::getWallets() const return m_wallets; } +std::vector<std::string> WalletController::getWalletsAvailableToOpen() const +{ + QMutexLocker locker(&m_mutex); + std::vector<std::string> wallets = m_node.listWalletDir(); + for (WalletModel* wallet_model : m_wallets) { + auto it = std::remove(wallets.begin(), wallets.end(), wallet_model->wallet().getWalletName()); + if (it != wallets.end()) wallets.erase(it); + } + return wallets; +} + +OpenWalletActivity* WalletController::openWallet(const std::string& name, QWidget* parent) +{ + OpenWalletActivity* activity = new OpenWalletActivity(this, name); + activity->moveToThread(&m_activity_thread); + return activity; +} + +void WalletController::closeWallet(WalletModel* wallet_model, QWidget* parent) +{ + QMessageBox box(parent); + box.setWindowTitle(tr("Close wallet")); + box.setText(tr("Are you sure you wish to close wallet <i>%1</i>?").arg(wallet_model->getDisplayName())); + box.setInformativeText(tr("Closing the wallet for too long can result in having to resync the entire chain if pruning is enabled.")); + box.setStandardButtons(QMessageBox::Yes|QMessageBox::Cancel); + box.setDefaultButton(QMessageBox::Yes); + if (box.exec() != QMessageBox::Yes) return; + + // First remove wallet from node. + wallet_model->wallet().remove(); + // Now release the model. + removeAndDeleteWallet(wallet_model); +} + WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wallet> wallet) { QMutexLocker locker(&m_mutex); @@ -93,3 +134,24 @@ void WalletController::removeAndDeleteWallet(WalletModel* wallet_model) // CWallet shared pointer. delete wallet_model; } + + +OpenWalletActivity::OpenWalletActivity(WalletController* wallet_controller, const std::string& name) + : m_wallet_controller(wallet_controller) + , m_name(name) +{} + +void OpenWalletActivity::open() +{ + std::string error, warning; + std::unique_ptr<interfaces::Wallet> wallet = m_wallet_controller->m_node.loadWallet(m_name, error, warning); + if (!warning.empty()) { + Q_EMIT message(QMessageBox::Warning, QString::fromStdString(warning)); + } + if (wallet) { + Q_EMIT opened(m_wallet_controller->getOrCreateWallet(std::move(wallet))); + } else { + Q_EMIT message(QMessageBox::Critical, QString::fromStdString(error)); + } + Q_EMIT finished(); +} diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index 22b71b07ff..19b3a82253 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -12,7 +12,9 @@ #include <memory> #include <vector> +#include <QMessageBox> #include <QMutex> +#include <QThread> class OptionsModel; class PlatformStyle; @@ -22,6 +24,8 @@ class Handler; class Node; } // namespace interfaces +class OpenWalletActivity; + /** * Controller between interfaces::Node, WalletModel instances and the GUI. */ @@ -37,6 +41,10 @@ public: ~WalletController(); std::vector<WalletModel*> getWallets() const; + std::vector<std::string> getWalletsAvailableToOpen() const; + + OpenWalletActivity* openWallet(const std::string& name, QWidget* parent = nullptr); + void closeWallet(WalletModel* wallet_model, QWidget* parent = nullptr); private Q_SLOTS: void addWallet(WalletModel* wallet_model); @@ -48,12 +56,35 @@ Q_SIGNALS: void coinsSent(WalletModel* wallet_model, SendCoinsRecipient recipient, QByteArray transaction); private: + QThread m_activity_thread; interfaces::Node& m_node; const PlatformStyle* const m_platform_style; OptionsModel* const m_options_model; mutable QMutex m_mutex; std::vector<WalletModel*> m_wallets; std::unique_ptr<interfaces::Handler> m_handler_load_wallet; + + friend class OpenWalletActivity; +}; + +class OpenWalletActivity : public QObject +{ + Q_OBJECT + +public: + OpenWalletActivity(WalletController* wallet_controller, const std::string& name); + +public Q_SLOTS: + void open(); + +Q_SIGNALS: + void message(QMessageBox::Icon icon, const QString text); + void finished(); + void opened(WalletModel* wallet_model); + +private: + WalletController* const m_wallet_controller; + std::string const m_name; }; #endif // BITCOIN_QT_WALLETCONTROLLER_H diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 466f2278eb..94413547d4 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -40,15 +40,11 @@ void WalletFrame::setClientModel(ClientModel *_clientModel) this->clientModel = _clientModel; } -bool WalletFrame::addWallet(WalletModel *walletModel) +void WalletFrame::addWallet(WalletModel *walletModel) { - if (!gui || !clientModel || !walletModel) { - return false; - } + if (!gui || !clientModel || !walletModel) return; - if (mapWalletViews.count(walletModel) > 0) { - return false; - } + if (mapWalletViews.count(walletModel) > 0) return; WalletView *walletView = new WalletView(platformStyle, this); walletView->setBitcoinGUI(gui); @@ -72,31 +68,25 @@ bool WalletFrame::addWallet(WalletModel *walletModel) }); connect(walletView, &WalletView::outOfSyncWarningClicked, this, &WalletFrame::outOfSyncWarningClicked); - - return true; } -bool WalletFrame::setCurrentWallet(WalletModel* wallet_model) +void WalletFrame::setCurrentWallet(WalletModel* wallet_model) { - if (mapWalletViews.count(wallet_model) == 0) - return false; + if (mapWalletViews.count(wallet_model) == 0) return; WalletView *walletView = mapWalletViews.value(wallet_model); walletStack->setCurrentWidget(walletView); assert(walletView); walletView->updateEncryptionStatus(); - return true; } -bool WalletFrame::removeWallet(WalletModel* wallet_model) +void WalletFrame::removeWallet(WalletModel* wallet_model) { - if (mapWalletViews.count(wallet_model) == 0) - return false; + if (mapWalletViews.count(wallet_model) == 0) return; WalletView *walletView = mapWalletViews.take(wallet_model); walletStack->removeWidget(walletView); delete walletView; - return true; } void WalletFrame::removeAllWallets() diff --git a/src/qt/walletframe.h b/src/qt/walletframe.h index 6a74fde9fd..156653f47d 100644 --- a/src/qt/walletframe.h +++ b/src/qt/walletframe.h @@ -36,9 +36,9 @@ public: void setClientModel(ClientModel *clientModel); - bool addWallet(WalletModel *walletModel); - bool setCurrentWallet(WalletModel* wallet_model); - bool removeWallet(WalletModel* wallet_model); + void addWallet(WalletModel *walletModel); + void setCurrentWallet(WalletModel* wallet_model); + void removeWallet(WalletModel* wallet_model); void removeAllWallets(); bool handlePaymentRequest(const SendCoinsRecipient& recipient); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 2a9144bec9..f4f3be8f43 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -580,12 +580,7 @@ bool WalletModel::privateKeysDisabled() const bool WalletModel::canGetAddresses() const { - // The wallet can provide a fresh address if: - // * hdEnabled(): an HD seed is present; or - // * it is a legacy wallet, because: - // * !hdEnabled(): an HD seed is not present; and - // * !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS): private keys have not been disabled (which results in hdEnabled() == true) - return m_wallet->hdEnabled() || (!m_wallet->hdEnabled() && !m_wallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + return m_wallet->canGetAddresses(); } QString WalletModel::getWalletName() const diff --git a/src/random.cpp b/src/random.cpp index 3277c34d3f..1aa78a9034 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -78,25 +78,119 @@ static inline int64_t GetPerformanceCounter() noexcept } #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) -static bool rdrand_supported = false; +static bool g_rdrand_supported = false; +static bool g_rdseed_supported = false; static constexpr uint32_t CPUID_F1_ECX_RDRAND = 0x40000000; +static constexpr uint32_t CPUID_F7_EBX_RDSEED = 0x00040000; +#ifdef bit_RDRND +static_assert(CPUID_F1_ECX_RDRAND == bit_RDRND, "Unexpected value for bit_RDRND"); +#endif +#ifdef bit_RDSEED +static_assert(CPUID_F7_EBX_RDSEED == bit_RDSEED, "Unexpected value for bit_RDSEED"); +#endif +static void inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) +{ + // We can't use __get_cpuid as it doesn't support subleafs. +#ifdef __GNUC__ + __cpuid_count(leaf, subleaf, a, b, c, d); +#else + __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf)); +#endif +} + static void InitHardwareRand() { uint32_t eax, ebx, ecx, edx; - if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx & CPUID_F1_ECX_RDRAND)) { - rdrand_supported = true; + GetCPUID(1, 0, eax, ebx, ecx, edx); + if (ecx & CPUID_F1_ECX_RDRAND) { + g_rdrand_supported = true; + } + GetCPUID(7, 0, eax, ebx, ecx, edx); + if (ebx & CPUID_F7_EBX_RDSEED) { + g_rdseed_supported = true; } } static void ReportHardwareRand() { - if (rdrand_supported) { - // This must be done in a separate function, as HWRandInit() may be indirectly called - // from global constructors, before logging is initialized. + // This must be done in a separate function, as HWRandInit() may be indirectly called + // from global constructors, before logging is initialized. + if (g_rdseed_supported) { + LogPrintf("Using RdSeed as additional entropy source\n"); + } + if (g_rdrand_supported) { LogPrintf("Using RdRand as an additional entropy source\n"); } } +/** Read 64 bits of entropy using rdrand. + * + * Must only be called when RdRand is supported. + */ +static uint64_t GetRdRand() noexcept +{ + // RdRand may very rarely fail. Invoke it up to 10 times in a loop to reduce this risk. +#ifdef __i386__ + uint8_t ok; + uint32_t r1, r2; + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %eax + if (ok) break; + } + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r2), "=q"(ok) :: "cc"); // rdrand %eax + if (ok) break; + } + return (((uint64_t)r2) << 32) | r1; +#elif defined(__x86_64__) || defined(__amd64__) + uint8_t ok; + uint64_t r1; + for (int i = 0; i < 10; ++i) { + __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf0; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdrand %rax + if (ok) break; + } + return r1; +#else +#error "RdRand is only supported on x86 and x86_64" +#endif +} + +/** Read 64 bits of entropy using rdseed. + * + * Must only be called when RdSeed is supported. + */ +static uint64_t GetRdSeed() noexcept +{ + // RdSeed may fail when the HW RNG is overloaded. Loop indefinitely until enough entropy is gathered, + // but pause after every failure. +#ifdef __i386__ + uint8_t ok; + uint32_t r1, r2; + do { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdseed %eax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + do { + __asm__ volatile (".byte 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r2), "=q"(ok) :: "cc"); // rdseed %eax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + return (((uint64_t)r2) << 32) | r1; +#elif defined(__x86_64__) || defined(__amd64__) + uint8_t ok; + uint64_t r1; + do { + __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf8; setc %1" : "=a"(r1), "=q"(ok) :: "cc"); // rdseed %rax + if (ok) break; + __asm__ volatile ("pause"); + } while(true); + return r1; +#else +#error "RdSeed is only supported on x86 and x86_64" +#endif +} + #else /* Access to other hardware random number generators could be added here later, * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). @@ -107,40 +201,40 @@ static void InitHardwareRand() {} static void ReportHardwareRand() {} #endif -static bool GetHardwareRand(unsigned char* ent32) noexcept { +/** Add 64 bits of entropy gathered from hardware to hasher. Do nothing if not supported. */ +static void SeedHardwareFast(CSHA512& hasher) noexcept { #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) - if (rdrand_supported) { - uint8_t ok; - // Not all assemblers support the rdrand instruction, write it in hex. -#ifdef __i386__ - for (int iter = 0; iter < 4; ++iter) { - uint32_t r1, r2; - __asm__ volatile (".byte 0x0f, 0xc7, 0xf0;" // rdrand %eax - ".byte 0x0f, 0xc7, 0xf2;" // rdrand %edx - "setc %2" : - "=a"(r1), "=d"(r2), "=q"(ok) :: "cc"); - if (!ok) return false; - WriteLE32(ent32 + 8 * iter, r1); - WriteLE32(ent32 + 8 * iter + 4, r2); - } -#else - uint64_t r1, r2, r3, r4; - __asm__ volatile (".byte 0x48, 0x0f, 0xc7, 0xf0, " // rdrand %rax - "0x48, 0x0f, 0xc7, 0xf3, " // rdrand %rbx - "0x48, 0x0f, 0xc7, 0xf1, " // rdrand %rcx - "0x48, 0x0f, 0xc7, 0xf2; " // rdrand %rdx - "setc %4" : - "=a"(r1), "=b"(r2), "=c"(r3), "=d"(r4), "=q"(ok) :: "cc"); - if (!ok) return false; - WriteLE64(ent32, r1); - WriteLE64(ent32 + 8, r2); - WriteLE64(ent32 + 16, r3); - WriteLE64(ent32 + 24, r4); + if (g_rdrand_supported) { + uint64_t out = GetRdRand(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + return; + } #endif - return true; +} + +/** Add 256 bits of entropy gathered from hardware to hasher. Do nothing if not supported. */ +static void SeedHardwareSlow(CSHA512& hasher) noexcept { +#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__) + // When we want 256 bits of entropy, prefer RdSeed over RdRand, as it's + // guaranteed to produce independent randomness on every call. + if (g_rdseed_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = GetRdSeed(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; + } + // When falling back to RdRand, XOR the result of 1024 results. + // This guarantees a reseeding occurs between each. + if (g_rdrand_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = 0; + for (int j = 0; j < 1024; ++j) out ^= GetRdRand(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; } #endif - return false; } static void RandAddSeedPerfmon(CSHA512& hasher) @@ -407,8 +501,7 @@ static void SeedFast(CSHA512& hasher) noexcept hasher.Write((const unsigned char*)&ptr, sizeof(ptr)); // Hardware randomness is very fast when available; use it always. - bool have_hw_rand = GetHardwareRand(buffer); - if (have_hw_rand) hasher.Write(buffer, sizeof(buffer)); + SeedHardwareFast(hasher); // High-precision timestamp SeedTimestamp(hasher); @@ -460,6 +553,9 @@ static void SeedStartup(CSHA512& hasher) noexcept RAND_screen(); #endif + // Gather 256 bits of hardware randomness, if available + SeedHardwareSlow(hasher); + // Everything that the 'slow' seeder includes. SeedSlow(hasher); diff --git a/src/random.h b/src/random.h index 4c73f3822a..1c035f87ba 100644 --- a/src/random.h +++ b/src/random.h @@ -24,7 +24,7 @@ * perform 'fast' seeding, consisting of mixing in: * - A stack pointer (indirectly committing to calling thread and call stack) * - A high-precision timestamp (rdtsc when available, c++ high_resolution_clock otherwise) - * - Hardware RNG (rdrand) when available. + * - 64 bits from the hardware RNG (rdrand) when available. * These entropy sources are very fast, and only designed to protect against situations * where a VM state restore/copy results in multiple systems with the same randomness. * FastRandomContext on the other hand does not protect against this once created, but @@ -48,6 +48,7 @@ * * On first use of the RNG (regardless of what function is called first), all entropy * sources used in the 'slow' seeder are included, but also: + * - 256 bits from the hardware RNG (rdseed or rdrand) when available. * - (On Windows) Performance monitoring data from the OS. * - (On Windows) Through OpenSSL, the screen contents. * diff --git a/src/rest.cpp b/src/rest.cpp index 326f7ae1d2..baad3b2ce9 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -300,7 +300,7 @@ static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) switch (rf) { case RetFormat::JSON: { - UniValue mempoolInfoObject = mempoolInfoToJSON(); + UniValue mempoolInfoObject = MempoolInfoToJSON(::mempool); std::string strJSON = mempoolInfoObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); @@ -322,7 +322,7 @@ static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPar switch (rf) { case RetFormat::JSON: { - UniValue mempoolObject = mempoolToJSON(true); + UniValue mempoolObject = MempoolToJSON(::mempool, true); std::string strJSON = mempoolObject.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index fa0b62ec48..1827aec637 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -215,7 +215,7 @@ static UniValue waitfornewblock(const JSONRPCRequest& request) "\nWaits for a specific new block and returns useful info about it.\n" "\nReturns the current block on timeout or exit.\n", { - {"timeout", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "Time in milliseconds to wait for a response. 0 indicates no timeout."}, + {"timeout", RPCArg::Type::NUM, /* default */ "0", "Time in milliseconds to wait for a response. 0 indicates no timeout."}, }, RPCResult{ "{ (json object)\n" @@ -256,8 +256,8 @@ static UniValue waitforblock(const JSONRPCRequest& request) "\nWaits for a specific new block and returns useful info about it.\n" "\nReturns the current block on timeout or exit.\n", { - {"blockhash", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "Block hash to wait for."}, - {"timeout", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "Time in milliseconds to wait for a response. 0 indicates no timeout."}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "Block hash to wait for."}, + {"timeout", RPCArg::Type::NUM, /* default */ "0", "Time in milliseconds to wait for a response. 0 indicates no timeout."}, }, RPCResult{ "{ (json object)\n" @@ -302,8 +302,8 @@ static UniValue waitforblockheight(const JSONRPCRequest& request) "of the current tip.\n" "\nReturns the current block on timeout or exit.\n", { - {"height", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "Block height to wait for."}, - {"timeout", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "Time in milliseconds to wait for a response. 0 indicates no timeout."}, + {"height", RPCArg::Type::NUM, RPCArg::Optional::NO, "Block height to wait for."}, + {"timeout", RPCArg::Type::NUM, /* default */ "0", "Time in milliseconds to wait for a response. 0 indicates no timeout."}, }, RPCResult{ "{ (json object)\n" @@ -405,9 +405,9 @@ static std::string EntryDescriptionString() " \"bip125-replaceable\" : true|false, (boolean) Whether this transaction could be replaced due to BIP125 (replace-by-fee)\n"; } -static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCKS_REQUIRED(::mempool.cs) +static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPoolEntry& e) EXCLUSIVE_LOCKS_REQUIRED(pool.cs) { - AssertLockHeld(mempool.cs); + AssertLockHeld(pool.cs); UniValue fees(UniValue::VOBJ); fees.pushKV("base", ValueFromAmount(e.GetFee())); @@ -427,12 +427,12 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK info.pushKV("ancestorcount", e.GetCountWithAncestors()); info.pushKV("ancestorsize", e.GetSizeWithAncestors()); info.pushKV("ancestorfees", e.GetModFeesWithAncestors()); - info.pushKV("wtxid", mempool.vTxHashes[e.vTxHashesIdx].first.ToString()); + info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); const CTransaction& tx = e.GetTx(); std::set<std::string> setDepends; for (const CTxIn& txin : tx.vin) { - if (mempool.exists(txin.prevout.hash)) + if (pool.exists(txin.prevout.hash)) setDepends.insert(txin.prevout.hash.ToString()); } @@ -445,8 +445,8 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK info.pushKV("depends", depends); UniValue spent(UniValue::VARR); - const CTxMemPool::txiter &it = mempool.mapTx.find(tx.GetHash()); - const CTxMemPool::setEntries &setChildren = mempool.GetMemPoolChildren(it); + const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); + const CTxMemPool::setEntries& setChildren = pool.GetMemPoolChildren(it); for (CTxMemPool::txiter childiter : setChildren) { spent.push_back(childiter->GetTx().GetHash().ToString()); } @@ -455,7 +455,7 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK // Add opt-in RBF status bool rbfStatus = false; - RBFTransactionState rbfState = IsRBFOptIn(tx, mempool); + RBFTransactionState rbfState = IsRBFOptIn(tx, pool); if (rbfState == RBFTransactionState::UNKNOWN) { throw JSONRPCError(RPC_MISC_ERROR, "Transaction is not in mempool"); } else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) { @@ -465,25 +465,21 @@ static void entryToJSON(UniValue &info, const CTxMemPoolEntry &e) EXCLUSIVE_LOCK info.pushKV("bip125-replaceable", rbfStatus); } -UniValue mempoolToJSON(bool fVerbose) +UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose) { - if (fVerbose) - { - LOCK(mempool.cs); + if (verbose) { + LOCK(pool.cs); UniValue o(UniValue::VOBJ); - for (const CTxMemPoolEntry& e : mempool.mapTx) - { + for (const CTxMemPoolEntry& e : pool.mapTx) { const uint256& hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(pool, info, e); o.pushKV(hash.ToString(), info); } return o; - } - else - { + } else { std::vector<uint256> vtxid; - mempool.queryHashes(vtxid); + pool.queryHashes(vtxid); UniValue a(UniValue::VARR); for (const uint256& hash : vtxid) @@ -501,7 +497,7 @@ static UniValue getrawmempool(const JSONRPCRequest& request) "\nReturns all transaction ids in memory pool as a json array of string transaction ids.\n" "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n", { - {"verbose", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "True for a json object, false for array of transaction ids"}, + {"verbose", RPCArg::Type::BOOL, /* default */ "false", "True for a json object, false for array of transaction ids"}, }, RPCResult{"for verbose = false", "[ (json array of string)\n" @@ -525,7 +521,7 @@ static UniValue getrawmempool(const JSONRPCRequest& request) if (!request.params[0].isNull()) fVerbose = request.params[0].get_bool(); - return mempoolToJSON(fVerbose); + return MempoolToJSON(::mempool, fVerbose); } static UniValue getmempoolancestors(const JSONRPCRequest& request) @@ -535,8 +531,8 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) RPCHelpMan{"getmempoolancestors", "\nIf txid is in the mempool, returns all in-mempool ancestors.\n", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id (must be in mempool)"}, - {"verbose", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "True for a json object, false for array of transaction ids"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, + {"verbose", RPCArg::Type::BOOL, /* default */ "false", "True for a json object, false for array of transaction ids"}, }, { RPCResult{"for verbose = false", @@ -591,7 +587,7 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *ancestorIt; const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::mempool, info, e); o.pushKV(_hash.ToString(), info); } return o; @@ -605,8 +601,8 @@ static UniValue getmempooldescendants(const JSONRPCRequest& request) RPCHelpMan{"getmempooldescendants", "\nIf txid is in the mempool, returns all in-mempool descendants.\n", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id (must be in mempool)"}, - {"verbose", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "True for a json object, false for array of transaction ids"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, + {"verbose", RPCArg::Type::BOOL, /* default */ "false", "True for a json object, false for array of transaction ids"}, }, { RPCResult{"for verbose = false", @@ -661,7 +657,7 @@ static UniValue getmempooldescendants(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *descendantIt; const uint256& _hash = e.GetTx().GetHash(); UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::mempool, info, e); o.pushKV(_hash.ToString(), info); } return o; @@ -675,7 +671,7 @@ static UniValue getmempoolentry(const JSONRPCRequest& request) RPCHelpMan{"getmempoolentry", "\nReturns mempool data for given transaction\n", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id (must be in mempool)"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id (must be in mempool)"}, }, RPCResult{ "{ (json object)\n" @@ -700,7 +696,7 @@ static UniValue getmempoolentry(const JSONRPCRequest& request) const CTxMemPoolEntry &e = *it; UniValue info(UniValue::VOBJ); - entryToJSON(info, e); + entryToJSON(::mempool, info, e); return info; } @@ -711,7 +707,7 @@ static UniValue getblockhash(const JSONRPCRequest& request) RPCHelpMan{"getblockhash", "\nReturns hash of block in best-block-chain at height provided.\n", { - {"height", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The height index"}, + {"height", RPCArg::Type::NUM, RPCArg::Optional::NO, "The height index"}, }, RPCResult{ "\"hash\" (string) The block hash\n" @@ -740,8 +736,8 @@ static UniValue getblockheader(const JSONRPCRequest& request) "\nIf verbose is false, returns a string that is serialized, hex-encoded data for blockheader 'hash'.\n" "If verbose is true, returns an Object with information about blockheader <hash>.\n", { - {"blockhash", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The block hash"}, - {"verbose", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "true for a json object, false for the hex-encoded data"}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, + {"verbose", RPCArg::Type::BOOL, /* default */ "true", "true for a json object, false for the hex-encoded data"}, }, { RPCResult{"for verbose = true", @@ -830,8 +826,8 @@ static UniValue getblock(const JSONRPCRequest& request) "If verbosity is 1, returns an Object with information about block <hash>.\n" "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n", { - {"blockhash", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The block hash"}, - {"verbosity", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1", "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, + {"verbosity", RPCArg::Type::NUM, /* default */ "1", "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, }, { RPCResult{"for verbosity = 0", @@ -986,7 +982,7 @@ static UniValue pruneblockchain(const JSONRPCRequest& request) throw std::runtime_error( RPCHelpMan{"pruneblockchain", "", { - {"height", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The block height to prune up to. May be set to a discrete height, or a unix timestamp\n" + {"height", RPCArg::Type::NUM, RPCArg::Optional::NO, "The block height to prune up to. May be set to a discrete height, or a unix timestamp\n" " to prune blocks whose block time is at least 2 hours older than the provided timestamp."}, }, RPCResult{ @@ -1085,9 +1081,9 @@ UniValue gettxout(const JSONRPCRequest& request) RPCHelpMan{"gettxout", "\nReturns details about an unspent transaction output.\n", { - {"txid", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The transaction id"}, - {"n", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "vout number"}, - {"include_mempool", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "Whether to include the mempool. Note that an unspent output that is spent in the mempool won't appear."}, + {"txid", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction id"}, + {"n", RPCArg::Type::NUM, RPCArg::Optional::NO, "vout number"}, + {"include_mempool", RPCArg::Type::BOOL, /* default */ "true", "Whether to include the mempool. Note that an unspent output that is spent in the mempool won't appear."}, }, RPCResult{ "{\n" @@ -1166,8 +1162,8 @@ static UniValue verifychain(const JSONRPCRequest& request) RPCHelpMan{"verifychain", "\nVerifies blockchain database.\n", { - {"checklevel", RPCArg::Type::NUM, /* opt */ true, /* default_val */ strprintf("%d, range=0-4", nCheckLevel), "How thorough the block verification is."}, - {"nblocks", RPCArg::Type::NUM, /* opt */ true, /* default_val */ strprintf("%d, 0=all", nCheckDepth), "The number of blocks to check."}, + {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", nCheckLevel), "How thorough the block verification is."}, + {"nblocks", RPCArg::Type::NUM, /* default */ strprintf("%d, 0=all", nCheckDepth), "The number of blocks to check."}, }, RPCResult{ "true|false (boolean) Verified or not\n" @@ -1485,15 +1481,15 @@ static UniValue getchaintips(const JSONRPCRequest& request) return res; } -UniValue mempoolInfoToJSON() +UniValue MempoolInfoToJSON(const CTxMemPool& pool) { UniValue ret(UniValue::VOBJ); - ret.pushKV("size", (int64_t) mempool.size()); - ret.pushKV("bytes", (int64_t) mempool.GetTotalTxSize()); - ret.pushKV("usage", (int64_t) mempool.DynamicMemoryUsage()); + ret.pushKV("size", (int64_t)pool.size()); + ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); + ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); size_t maxmempool = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; ret.pushKV("maxmempool", (int64_t) maxmempool); - ret.pushKV("mempoolminfee", ValueFromAmount(std::max(mempool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); + ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); return ret; @@ -1522,7 +1518,7 @@ static UniValue getmempoolinfo(const JSONRPCRequest& request) }, }.ToString()); - return mempoolInfoToJSON(); + return MempoolInfoToJSON(::mempool); } static UniValue preciousblock(const JSONRPCRequest& request) @@ -1534,7 +1530,7 @@ static UniValue preciousblock(const JSONRPCRequest& request) "\nA later preciousblock call can override the effect of an earlier one.\n" "\nThe effects of preciousblock are not retained across restarts.\n", { - {"blockhash", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "the hash of the block to mark as precious"}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hash of the block to mark as precious"}, }, RPCResults{}, RPCExamples{ @@ -1571,7 +1567,7 @@ static UniValue invalidateblock(const JSONRPCRequest& request) RPCHelpMan{"invalidateblock", "\nPermanently marks a block as invalid, as if it violated a consensus rule.\n", { - {"blockhash", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "the hash of the block to mark as invalid"}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hash of the block to mark as invalid"}, }, RPCResults{}, RPCExamples{ @@ -1583,15 +1579,15 @@ static UniValue invalidateblock(const JSONRPCRequest& request) uint256 hash(ParseHashV(request.params[0], "blockhash")); CValidationState state; + CBlockIndex* pblockindex; { LOCK(cs_main); - CBlockIndex* pblockindex = LookupBlockIndex(hash); + pblockindex = LookupBlockIndex(hash); if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } - - InvalidateBlock(state, Params(), pblockindex); } + InvalidateBlock(state, Params(), pblockindex); if (state.IsValid()) { ActivateBestChain(state, Params()); @@ -1612,7 +1608,7 @@ static UniValue reconsiderblock(const JSONRPCRequest& request) "\nRemoves invalidity status of a block, its ancestors and its descendants, reconsider them for activation.\n" "This can be used to undo the effects of invalidateblock.\n", { - {"blockhash", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "the hash of the block to reconsider"}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hash of the block to reconsider"}, }, RPCResults{}, RPCExamples{ @@ -1650,8 +1646,8 @@ static UniValue getchaintxstats(const JSONRPCRequest& request) RPCHelpMan{"getchaintxstats", "\nCompute statistics about the total number and rate of transactions in the chain.\n", { - {"nblocks", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "one month", "Size of the window in number of blocks"}, - {"blockhash", RPCArg::Type::STR_HEX, /* opt */ true, /* default_val */ "chain tip", "The hash of the block that ends the window."}, + {"nblocks", RPCArg::Type::NUM, /* default */ "one month", "Size of the window in number of blocks"}, + {"blockhash", RPCArg::Type::STR_HEX, /* default */ "chain tip", "The hash of the block that ends the window."}, }, RPCResult{ "{\n" @@ -1778,18 +1774,16 @@ static constexpr size_t PER_UTXO_OVERHEAD = sizeof(COutPoint) + sizeof(uint32_t) static UniValue getblockstats(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 4) { - throw std::runtime_error( - RPCHelpMan{"getblockstats", + const RPCHelpMan help{"getblockstats", "\nCompute per block statistics for a given window. All amounts are in satoshis.\n" "It won't work for some heights with pruning.\n" "It won't work without -txindex for utxo_size_inc, *fee or *feerate stats.\n", { - {"hash_or_height", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The block hash or height of the target block", "", {"", "string or numeric"}}, - {"stats", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "all values", "Values to plot (see result below)", + {"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::NO, "The block hash or height of the target block", "", {"", "string or numeric"}}, + {"stats", RPCArg::Type::ARR, /* default */ "all values", "Values to plot (see result below)", { - {"height", RPCArg::Type::STR, /* opt */ true, /* default_val */ "", "Selected statistic"}, - {"time", RPCArg::Type::STR, /* opt */ true, /* default_val */ "", "Selected statistic"}, + {"height", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Selected statistic"}, + {"time", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Selected statistic"}, }, "stats"}, }, @@ -1836,7 +1830,9 @@ static UniValue getblockstats(const JSONRPCRequest& request) HelpExampleCli("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") + HelpExampleRpc("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") }, - }.ToString()); + }; + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); } LOCK(cs_main); @@ -2148,18 +2144,18 @@ UniValue scantxoutset(const JSONRPCRequest& request) "In the latter case, a range needs to be specified by below if different from 1000.\n" "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n", { - {"action", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The action to execute\n" + {"action", RPCArg::Type::STR, RPCArg::Optional::NO, "The action to execute\n" " \"start\" for starting a scan\n" " \"abort\" for aborting the current scan (returns true when abort was successful)\n" " \"status\" for progress report (in %) of the current scan"}, - {"scanobjects", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "Array of scan objects\n" + {"scanobjects", RPCArg::Type::ARR, RPCArg::Optional::NO, "Array of scan objects\n" " Every scan object is either a string descriptor or an object:", { - {"descriptor", RPCArg::Type::STR, /* opt */ true, /* default_val */ "", "An output descriptor"}, - {"", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "An object with output descriptor and metadata", + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with output descriptor and metadata", { - {"desc", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "An output descriptor"}, - {"range", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1000", "Up to what child index HD chains should be explored"}, + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, + {"range", RPCArg::Type::RANGE, /* default */ "1000", "The range of HD chain indexes to explore (either end or [begin,end])"}, }, }, }, @@ -2216,7 +2212,7 @@ UniValue scantxoutset(const JSONRPCRequest& request) // loop through the scan objects for (const UniValue& scanobject : request.params[1].get_array().getValues()) { std::string desc_str; - int range = 1000; + std::pair<int64_t, int64_t> range = {0, 1000}; if (scanobject.isStr()) { desc_str = scanobject.get_str(); } else if (scanobject.isObject()) { @@ -2225,8 +2221,8 @@ UniValue scantxoutset(const JSONRPCRequest& request) desc_str = desc_uni.get_str(); UniValue range_uni = find_value(scanobject, "range"); if (!range_uni.isNull()) { - range = range_uni.get_int(); - if (range < 0 || range > 1000000) throw JSONRPCError(RPC_INVALID_PARAMETER, "range out of range"); + range = ParseRange(range_uni); + if (range.first < 0 || (range.second >> 31) != 0 || range.second >= range.first + 1000000) throw JSONRPCError(RPC_INVALID_PARAMETER, "range out of range"); } } else { throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan object needs to be either a string or an object"); @@ -2237,8 +2233,11 @@ UniValue scantxoutset(const JSONRPCRequest& request) if (!desc) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Invalid descriptor '%s'", desc_str)); } - if (!desc->IsRange()) range = 0; - for (int i = 0; i <= range; ++i) { + if (!desc->IsRange()) { + range.first = 0; + range.second = 0; + } + for (int i = range.first; i <= range.second; ++i) { std::vector<CScript> scripts; if (!desc->Expand(i, provider, scripts, provider)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys: '%s'", desc_str)); diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 529132d033..55d1de453f 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -11,6 +11,7 @@ class CBlock; class CBlockIndex; +class CTxMemPool; class UniValue; static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; @@ -30,10 +31,10 @@ void RPCNotifyBlockChange(bool ibd, const CBlockIndex *); UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails = false); /** Mempool information to JSON */ -UniValue mempoolInfoToJSON(); +UniValue MempoolInfoToJSON(const CTxMemPool& pool); /** Mempool to JSON */ -UniValue mempoolToJSON(bool fVerbose = false); +UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false); /** Block header to JSON */ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index f95e22574c..a266580b3d 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -28,8 +28,6 @@ public: static const CRPCConvertParam vRPCConvertParams[] = { { "setmocktime", 0, "timestamp" }, - { "generate", 0, "nblocks" }, - { "generate", 1, "maxtries" }, { "generatetoaddress", 0, "nblocks" }, { "generatetoaddress", 2, "maxtries" }, { "getnetworkhashps", 0, "nblocks" }, @@ -68,6 +66,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendmany", 4, "subtractfeefrom" }, { "sendmany", 5 , "replaceable" }, { "sendmany", 6 , "conf_target" }, + { "deriveaddresses", 1, "range" }, { "scantxoutset", 1, "scanobjects" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, @@ -110,6 +109,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createpsbt", 2, "locktime" }, { "createpsbt", 3, "replaceable" }, { "combinepsbt", 0, "txs"}, + { "joinpsbts", 0, "txs"}, { "finalizepsbt", 1, "extract"}, { "converttopsbt", 1, "permitsigdata"}, { "converttopsbt", 2, "iswitness"}, @@ -159,6 +159,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "rescanblockchain", 0, "start_height"}, { "rescanblockchain", 1, "stop_height"}, { "createwallet", 1, "disable_private_keys"}, + { "createwallet", 2, "blank"}, { "getnodeaddresses", 0, "count"}, { "stop", 0, "wait" }, }; diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 200dfa107b..f2acb8fbf5 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -31,16 +31,6 @@ #include <memory> #include <stdint.h> -unsigned int ParseConfirmTarget(const UniValue& value) -{ - int target = value.get_int(); - unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); - if (target < 1 || (unsigned int)target > max_target) { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid conf_target, must be between %u - %u", 1, max_target)); - } - return (unsigned int)target; -} - /** * Return average network hashes per second based on the last 'lookup' blocks, * or from the last difficulty change if 'lookup' is nonpositive. @@ -92,8 +82,8 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) "Pass in [blocks] to override # of blocks, -1 specifies since last difficulty change.\n" "Pass in [height] to estimate the network speed at the time when a certain block was found.\n", { - {"nblocks", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "120", "The number of blocks, or -1 for blocks since last difficulty change."}, - {"height", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "-1", "To estimate at the time of the given height."}, + {"nblocks", RPCArg::Type::NUM, /* default */ "120", "The number of blocks, or -1 for blocks since last difficulty change."}, + {"height", RPCArg::Type::NUM, /* default */ "-1", "To estimate at the time of the given height."}, }, RPCResult{ "x (numeric) Hashes per second estimated\n" @@ -163,9 +153,9 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) RPCHelpMan{"generatetoaddress", "\nMine blocks immediately to a specified address (before the RPC call returns)\n", { - {"nblocks", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "How many blocks are generated immediately."}, - {"address", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The address to send the newly generated bitcoin to."}, - {"maxtries", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1000000", "How many iterations to try."}, + {"nblocks", RPCArg::Type::NUM, RPCArg::Optional::NO, "How many blocks are generated immediately."}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The address to send the newly generated bitcoin to."}, + {"maxtries", RPCArg::Type::NUM, /* default */ "1000000", "How many iterations to try."}, }, RPCResult{ "[ blockhashes ] (array) hashes of blocks generated\n" @@ -197,36 +187,36 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) static UniValue getmininginfo(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() != 0) + if (request.fHelp || request.params.size() != 0) { throw std::runtime_error( RPCHelpMan{"getmininginfo", "\nReturns a json object containing mining-related information.", {}, RPCResult{ - "{\n" - " \"blocks\": nnn, (numeric) The current block\n" - " \"currentblockweight\": nnn, (numeric) The last block weight\n" - " \"currentblocktx\": nnn, (numeric) The last block transaction\n" - " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" - " \"networkhashps\": nnn, (numeric) The network hashes per second\n" - " \"pooledtx\": n (numeric) The size of the mempool\n" - " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" - " \"warnings\": \"...\" (string) any network and blockchain warnings\n" - "}\n" + "{\n" + " \"blocks\": nnn, (numeric) The current block\n" + " \"currentblockweight\": nnn, (numeric, optional) The block weight of the last assembled block (only present if a block was ever assembled)\n" + " \"currentblocktx\": nnn, (numeric, optional) The number of block transactions of the last assembled block (only present if a block was ever assembled)\n" + " \"difficulty\": xxx.xxxxx (numeric) The current difficulty\n" + " \"networkhashps\": nnn, (numeric) The network hashes per second\n" + " \"pooledtx\": n (numeric) The size of the mempool\n" + " \"chain\": \"xxxx\", (string) current network name as defined in BIP70 (main, test, regtest)\n" + " \"warnings\": \"...\" (string) any network and blockchain warnings\n" + "}\n" }, RPCExamples{ HelpExampleCli("getmininginfo", "") + HelpExampleRpc("getmininginfo", "") }, }.ToString()); - + } LOCK(cs_main); UniValue obj(UniValue::VOBJ); obj.pushKV("blocks", (int)chainActive.Height()); - obj.pushKV("currentblockweight", (uint64_t)nLastBlockWeight); - obj.pushKV("currentblocktx", (uint64_t)nLastBlockTx); + if (BlockAssembler::m_last_block_weight) obj.pushKV("currentblockweight", *BlockAssembler::m_last_block_weight); + if (BlockAssembler::m_last_block_num_txs) obj.pushKV("currentblocktx", *BlockAssembler::m_last_block_num_txs); obj.pushKV("difficulty", (double)GetDifficulty(chainActive.Tip())); obj.pushKV("networkhashps", getnetworkhashps(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); @@ -244,10 +234,10 @@ static UniValue prioritisetransaction(const JSONRPCRequest& request) RPCHelpMan{"prioritisetransaction", "Accepts the transaction into mined blocks at a higher (or lower) priority\n", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id."}, - {"dummy", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "null", "API-Compatibility for previous API. Must be zero or null.\n" + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id."}, + {"dummy", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "API-Compatibility for previous API. Must be zero or null.\n" " DEPRECATED. For forward compatibility use named arguments and omit this parameter."}, - {"fee_delta", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The fee value (in satoshis) to add (or subtract, if negative).\n" + {"fee_delta", RPCArg::Type::NUM, RPCArg::Optional::NO, "The fee value (in satoshis) to add (or subtract, if negative).\n" " Note, that this value is not a fee rate. It is a value to modify absolute fee of the TX.\n" " The fee is not actually paid, only the algorithm for selecting transactions into a block\n" " considers the transaction as it would have paid a higher (or lower) fee."}, @@ -316,17 +306,17 @@ static UniValue getblocktemplate(const JSONRPCRequest& request) " https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki#getblocktemplate_changes\n" " https://github.com/bitcoin/bips/blob/master/bip-0145.mediawiki\n", { - {"template_request", RPCArg::Type::OBJ, /* opt */ false, /* default_val */ "", "A json object in the following spec", + {"template_request", RPCArg::Type::OBJ, RPCArg::Optional::NO, "A json object in the following spec", { - {"mode", RPCArg::Type::STR, /* opt */ true, /* default_val */ "", "This must be set to \"template\", \"proposal\" (see BIP 23), or omitted"}, - {"capabilities", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "", "A list of strings", + {"mode", RPCArg::Type::STR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "This must be set to \"template\", \"proposal\" (see BIP 23), or omitted"}, + {"capabilities", RPCArg::Type::ARR, /* treat as named arg */ RPCArg::Optional::OMITTED_NAMED_ARG, "A list of strings", { - {"support", RPCArg::Type::STR, /* opt */ true, /* default_val */ "", "client side supported feature, 'longpoll', 'coinbasetxn', 'coinbasevalue', 'proposal', 'serverlist', 'workid'"}, + {"support", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported feature, 'longpoll', 'coinbasetxn', 'coinbasevalue', 'proposal', 'serverlist', 'workid'"}, }, }, - {"rules", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "A list of strings", + {"rules", RPCArg::Type::ARR, RPCArg::Optional::NO, "A list of strings", { - {"support", RPCArg::Type::STR, /* opt */ true, /* default_val */ "", "client side supported softfork deployment"}, + {"support", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "client side supported softfork deployment"}, }, }, }, @@ -716,8 +706,8 @@ static UniValue submitblock(const JSONRPCRequest& request) "\nAttempts to submit new block to network.\n" "See https://en.bitcoin.it/wiki/BIP_0022 for full specification.\n", { - {"hexdata", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "the hex-encoded block data to submit"}, - {"dummy", RPCArg::Type::STR, /* opt */ true, /* default_val */ "ignored", "dummy value, for compatibility with BIP22. This value is ignored."}, + {"hexdata", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hex-encoded block data to submit"}, + {"dummy", RPCArg::Type::STR, /* default */ "ignored", "dummy value, for compatibility with BIP22. This value is ignored."}, }, RPCResults{}, RPCExamples{ @@ -781,7 +771,7 @@ static UniValue submitheader(const JSONRPCRequest& request) "\nDecode the given hexdata as a header and submit it as a candidate chain tip if valid." "\nThrows when the header is invalid.\n", { - {"hexdata", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "the hex-encoded block header data"}, + {"hexdata", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hex-encoded block header data"}, }, RPCResult{ "None" @@ -823,8 +813,8 @@ static UniValue estimatesmartfee(const JSONRPCRequest& request) "for which the estimate is valid. Uses virtual transaction size as defined\n" "in BIP 141 (witness data is discounted).\n", { - {"conf_target", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "Confirmation target in blocks (1 - 1008)"}, - {"estimate_mode", RPCArg::Type::STR, /* opt */ true, /* default_val */ "CONSERVATIVE", "The fee estimate mode.\n" + {"conf_target", RPCArg::Type::NUM, RPCArg::Optional::NO, "Confirmation target in blocks (1 - 1008)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "CONSERVATIVE", "The fee estimate mode.\n" " Whether to return a more conservative estimate which also satisfies\n" " a longer history. A conservative estimate potentially returns a\n" " higher feerate and is more likely to be sufficient for the desired\n" @@ -853,7 +843,8 @@ static UniValue estimatesmartfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VSTR}); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); - unsigned int conf_target = ParseConfirmTarget(request.params[0]); + unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); bool conservative = true; if (!request.params[1].isNull()) { FeeEstimateMode fee_mode; @@ -890,8 +881,8 @@ static UniValue estimaterawfee(const JSONRPCRequest& request) "confirmation within conf_target blocks if possible. Uses virtual transaction size as\n" "defined in BIP 141 (witness data is discounted).\n", { - {"conf_target", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "Confirmation target in blocks (1 - 1008)"}, - {"threshold", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0.95", "The proportion of transactions in a given feerate range that must have been\n" + {"conf_target", RPCArg::Type::NUM, RPCArg::Optional::NO, "Confirmation target in blocks (1 - 1008)"}, + {"threshold", RPCArg::Type::NUM, /* default */ "0.95", "The proportion of transactions in a given feerate range that must have been\n" " confirmed within conf_target in order to consider those feerates as high enough and proceed to check\n" " lower buckets."}, }, @@ -925,7 +916,8 @@ static UniValue estimaterawfee(const JSONRPCRequest& request) RPCTypeCheck(request.params, {UniValue::VNUM, UniValue::VNUM}, true); RPCTypeCheckArgument(request.params[0], UniValue::VNUM); - unsigned int conf_target = ParseConfirmTarget(request.params[0]); + unsigned int max_target = ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); + unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); double threshold = 0.95; if (!request.params[1].isNull()) { threshold = request.params[1].get_real(); diff --git a/src/rpc/mining.h b/src/rpc/mining.h index 8d46273159..be9a973315 100644 --- a/src/rpc/mining.h +++ b/src/rpc/mining.h @@ -12,7 +12,4 @@ /** Generate blocks (mine) */ UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbaseScript, int nGenerate, uint64_t nMaxTries, bool keepScript); -/** Check bounds on a command line confirm target */ -unsigned int ParseConfirmTarget(const UniValue& value); - #endif diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 8850cf066b..822a0beef9 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -16,6 +16,7 @@ #include <rpc/blockchain.h> #include <rpc/server.h> #include <rpc/util.h> +#include <script/descriptor.h> #include <timedata.h> #include <util/system.h> #include <util/strencodings.h> @@ -39,7 +40,7 @@ static UniValue validateaddress(const JSONRPCRequest& request) "fields have moved to getaddressinfo and will only be shown here with -deprecatedrpc=validateaddress: ismine, iswatchonly,\n" "script, hex, pubkeys, sigsrequired, pubkey, addresses, embedded, iscompressed, account, timestamp, hdkeypath, kdmasterkeyid.\n", { - {"address", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The bitcoin address to validate"}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to validate"}, }, RPCResult{ "{\n" @@ -86,12 +87,12 @@ static UniValue createmultisig(const JSONRPCRequest& request) "\nCreates a multi-signature address with n signature of m keys required.\n" "It returns a json object with the address and redeemScript.\n", { - {"nrequired", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The number of required signatures out of the n keys."}, - {"keys", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "A json array of hex-encoded public keys.", + {"nrequired", RPCArg::Type::NUM, RPCArg::Optional::NO, "The number of required signatures out of the n keys."}, + {"keys", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of hex-encoded public keys.", { - {"key", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The hex-encoded public key"}, + {"key", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "The hex-encoded public key"}, }}, - {"address_type", RPCArg::Type::STR, /* opt */ true, /* default_val */ "legacy", "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"address_type", RPCArg::Type::STR, /* default */ "legacy", "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, }, RPCResult{ "{\n" @@ -142,6 +143,135 @@ static UniValue createmultisig(const JSONRPCRequest& request) return result; } +UniValue getdescriptorinfo(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) { + throw std::runtime_error( + RPCHelpMan{"getdescriptorinfo", + {"\nAnalyses a descriptor.\n"}, + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor."}, + }, + RPCResult{ + "{\n" + " \"descriptor\" : \"desc\", (string) The descriptor in canonical form, without private keys\n" + " \"isrange\" : true|false, (boolean) Whether the descriptor is ranged\n" + " \"issolvable\" : true|false, (boolean) Whether the descriptor is solvable\n" + " \"hasprivatekeys\" : true|false, (boolean) Whether the input descriptor contained at least one private key\n" + "}\n" + }, + RPCExamples{ + "Analyse a descriptor\n" + + HelpExampleCli("getdescriptorinfo", "\"wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)\"") + }}.ToString() + ); + } + + RPCTypeCheck(request.params, {UniValue::VSTR}); + + FlatSigningProvider provider; + auto desc = Parse(request.params[0].get_str(), provider); + if (!desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Invalid descriptor")); + } + + UniValue result(UniValue::VOBJ); + result.pushKV("descriptor", desc->ToString()); + result.pushKV("isrange", desc->IsRange()); + result.pushKV("issolvable", desc->IsSolvable()); + result.pushKV("hasprivatekeys", provider.keys.size() > 0); + return result; +} + +UniValue deriveaddresses(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.empty() || request.params.size() > 2) { + throw std::runtime_error( + RPCHelpMan{"deriveaddresses", + {"\nDerives one or more addresses corresponding to an output descriptor.\n" + "Examples of output descriptors are:\n" + " pkh(<pubkey>) P2PKH outputs for the given pubkey\n" + " wpkh(<pubkey>) Native segwit P2PKH outputs for the given pubkey\n" + " sh(multi(<n>,<pubkey>,<pubkey>,...)) P2SH-multisig outputs for the given threshold and pubkeys\n" + " raw(<hex script>) Outputs whose scriptPubKey equals the specified hex scripts\n" + "\nIn the above, <pubkey> either refers to a fixed public key in hexadecimal notation, or to an xpub/xprv optionally followed by one\n" + "or more path elements separated by \"/\", where \"h\" represents a hardened child key.\n" + "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n"}, + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::NO, "The descriptor."}, + {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED_NAMED_ARG, "If a ranged descriptor is used, this specifies the end or the range (in [begin,end] notation) to derive."}, + }, + RPCResult{ + "[ address ] (array) the derived addresses\n" + }, + RPCExamples{ + "First three native segwit receive addresses\n" + + HelpExampleCli("deriveaddresses", "\"wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#trd0mf0l\" \"[0,2]\"") + }}.ToString() + ); + } + + RPCTypeCheck(request.params, {UniValue::VSTR, UniValueType()}); // Range argument is checked later + const std::string desc_str = request.params[0].get_str(); + + int64_t range_begin = 0; + int64_t range_end = 0; + + if (request.params.size() >= 2 && !request.params[1].isNull()) { + auto range = ParseRange(request.params[1]); + if (range.first < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should be greater or equal than 0"); + } + if ((range.second >> 31) != 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "End of range is too high"); + } + if (range.second >= range.first + 1000000) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range is too large"); + } + range_begin = range.first; + range_end = range.second; + } + + FlatSigningProvider provider; + auto desc = Parse(desc_str, provider, /* require_checksum = */ true); + if (!desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Invalid descriptor")); + } + + if (!desc->IsRange() && request.params.size() > 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for an un-ranged descriptor"); + } + + if (desc->IsRange() && request.params.size() == 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range must be specified for a ranged descriptor"); + } + + UniValue addresses(UniValue::VARR); + + for (int i = range_begin; i <= range_end; ++i) { + std::vector<CScript> scripts; + if (!desc->Expand(i, provider, scripts, provider)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); + } + + for (const CScript &script : scripts) { + CTxDestination dest; + if (!ExtractDestination(script, dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Descriptor does not have a corresponding address")); + } + + addresses.push_back(EncodeDestination(dest)); + } + } + + // This should not be possible, but an assert seems overkill: + if (addresses.empty()) { + throw JSONRPCError(RPC_MISC_ERROR, "Unexpected empty result"); + } + + return addresses; +} + static UniValue verifymessage(const JSONRPCRequest& request) { if (request.fHelp || request.params.size() != 3) @@ -149,9 +279,9 @@ static UniValue verifymessage(const JSONRPCRequest& request) RPCHelpMan{"verifymessage", "\nVerify a signed message\n", { - {"address", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The bitcoin address to use for the signature."}, - {"signature", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The signature provided by the signer in base 64 encoding (see signmessage)."}, - {"message", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The message that was signed."}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to use for the signature."}, + {"signature", RPCArg::Type::STR, RPCArg::Optional::NO, "The signature provided by the signer in base 64 encoding (see signmessage)."}, + {"message", RPCArg::Type::STR, RPCArg::Optional::NO, "The message that was signed."}, }, RPCResult{ "true|false (boolean) If the signature is verified or not.\n" @@ -208,8 +338,8 @@ static UniValue signmessagewithprivkey(const JSONRPCRequest& request) RPCHelpMan{"signmessagewithprivkey", "\nSign a message with the private key of an address\n", { - {"privkey", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The private key to sign the message with."}, - {"message", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The message to create a signature of."}, + {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key to sign the message with."}, + {"message", RPCArg::Type::STR, RPCArg::Optional::NO, "The message to create a signature of."}, }, RPCResult{ "\"signature\" (string) The signature of the message encoded in base 64\n" @@ -250,7 +380,7 @@ static UniValue setmocktime(const JSONRPCRequest& request) RPCHelpMan{"setmocktime", "\nSet the local time to given timestamp (-regtest only)\n", { - {"timestamp", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "Unix seconds-since-epoch timestamp\n" + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, "Unix seconds-since-epoch timestamp\n" " Pass 0 to go back to using the system time."}, }, RPCResults{}, @@ -316,7 +446,7 @@ static UniValue getmemoryinfo(const JSONRPCRequest& request) RPCHelpMan{"getmemoryinfo", "Returns an object containing information about memory usage.\n", { - {"mode", RPCArg::Type::STR, /* opt */ true, /* default_val */ "\"stats\"", "determines what kind of information is returned.\n" + {"mode", RPCArg::Type::STR, /* default */ "\"stats\"", "determines what kind of information is returned.\n" " - \"stats\" returns general statistics about memory usage in the daemon.\n" " - \"mallocinfo\" returns an XML string describing low-level heap state (only available if compiled with glibc 2.10+)."}, }, @@ -393,13 +523,13 @@ UniValue logging(const JSONRPCRequest& request) " - \"none\", \"0\" : even if other logging categories are specified, ignore all of them.\n" , { - {"include", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "null", "A json array of categories to add debug logging", + {"include", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "A json array of categories to add debug logging", { - {"include_category", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "the valid logging category"}, + {"include_category", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "the valid logging category"}, }}, - {"exclude", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "null", "A json array of categories to remove debug logging", + {"exclude", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "A json array of categories to remove debug logging", { - {"exclude_category", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "the valid logging category"}, + {"exclude_category", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "the valid logging category"}, }}, }, RPCResult{ @@ -473,6 +603,8 @@ static const CRPCCommand commands[] = { "control", "logging", &logging, {"include", "exclude"}}, { "util", "validateaddress", &validateaddress, {"address"} }, { "util", "createmultisig", &createmultisig, {"nrequired","keys","address_type"} }, + { "util", "deriveaddresses", &deriveaddresses, {"descriptor", "range"} }, + { "util", "getdescriptorinfo", &getdescriptorinfo, {"descriptor"} }, { "util", "verifymessage", &verifymessage, {"address","signature","message"} }, { "util", "signmessagewithprivkey", &signmessagewithprivkey, {"privkey","message"} }, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index b50038b085..c7b3478f44 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -221,8 +221,8 @@ static UniValue addnode(const JSONRPCRequest& request) "Nodes added using addnode (or -connect) are protected from DoS disconnection and are not required to be\n" "full nodes/support SegWit as other outbound peers are (though such peers will not be synced from).\n", { - {"node", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The node (see getpeerinfo for nodes)"}, - {"command", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "'add' to add a node to the list, 'remove' to remove a node from the list, 'onetry' to try a connection to the node once"}, + {"node", RPCArg::Type::STR, RPCArg::Optional::NO, "The node (see getpeerinfo for nodes)"}, + {"command", RPCArg::Type::STR, RPCArg::Optional::NO, "'add' to add a node to the list, 'remove' to remove a node from the list, 'onetry' to try a connection to the node once"}, }, RPCResults{}, RPCExamples{ @@ -266,8 +266,8 @@ static UniValue disconnectnode(const JSONRPCRequest& request) "\nStrictly one out of 'address' and 'nodeid' can be provided to identify the node.\n" "\nTo disconnect by nodeid, either set 'address' to the empty string, or call using the named 'nodeid' argument only.\n", { - {"address", RPCArg::Type::STR, /* opt */ true, /* default_val */ "fallback to nodeid", "The IP address/port of the node"}, - {"nodeid", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "fallback to address", "The node ID (see getpeerinfo for node IDs)"}, + {"address", RPCArg::Type::STR, /* default */ "fallback to nodeid", "The IP address/port of the node"}, + {"nodeid", RPCArg::Type::NUM, /* default */ "fallback to address", "The node ID (see getpeerinfo for node IDs)"}, }, RPCResults{}, RPCExamples{ @@ -311,7 +311,7 @@ static UniValue getaddednodeinfo(const JSONRPCRequest& request) "\nReturns information about the given added node, or all added nodes\n" "(note that onetry addnodes are not listed here)\n", { - {"node", RPCArg::Type::STR, /* opt */ true, /* default_val */ "all nodes", "If provided, return information about this specific node, otherwise all nodes are returned."}, + {"node", RPCArg::Type::STR, /* default */ "all nodes", "If provided, return information about this specific node, otherwise all nodes are returned."}, }, RPCResult{ "[\n" @@ -523,19 +523,13 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) static UniValue setban(const JSONRPCRequest& request) { - std::string strCommand; - if (!request.params[1].isNull()) - strCommand = request.params[1].get_str(); - if (request.fHelp || request.params.size() < 2 || - (strCommand != "add" && strCommand != "remove")) - throw std::runtime_error( - RPCHelpMan{"setban", + const RPCHelpMan help{"setban", "\nAttempts to add or remove an IP/Subnet from the banned list.\n", { - {"subnet", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The IP/Subnet (see getpeerinfo for nodes IP) with an optional netmask (default is /32 = single IP)"}, - {"command", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "'add' to add an IP/Subnet to the list, 'remove' to remove an IP/Subnet from the list"}, - {"bantime", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "time in seconds how long (or until when if [absolute] is set) the IP is banned (0 or empty means using the default time of 24h which can also be overwritten by the -bantime startup argument)"}, - {"absolute", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "If set, the bantime must be an absolute timestamp in seconds since epoch (Jan 1 1970 GMT)"}, + {"subnet", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP/Subnet (see getpeerinfo for nodes IP) with an optional netmask (default is /32 = single IP)"}, + {"command", RPCArg::Type::STR, RPCArg::Optional::NO, "'add' to add an IP/Subnet to the list, 'remove' to remove an IP/Subnet from the list"}, + {"bantime", RPCArg::Type::NUM, /* default */ "0", "time in seconds how long (or until when if [absolute] is set) the IP is banned (0 or empty means using the default time of 24h which can also be overwritten by the -bantime startup argument)"}, + {"absolute", RPCArg::Type::BOOL, /* default */ "false", "If set, the bantime must be an absolute timestamp in seconds since epoch (Jan 1 1970 GMT)"}, }, RPCResults{}, RPCExamples{ @@ -543,7 +537,13 @@ static UniValue setban(const JSONRPCRequest& request) + HelpExampleCli("setban", "\"192.168.0.0/24\" \"add\"") + HelpExampleRpc("setban", "\"192.168.0.6\", \"add\", 86400") }, - }.ToString()); + }; + std::string strCommand; + if (!request.params[1].isNull()) + strCommand = request.params[1].get_str(); + if (request.fHelp || !help.IsValidNumArgs(request.params.size()) || (strCommand != "add" && strCommand != "remove")) { + throw std::runtime_error(help.ToString()); + } if (!g_banman) { throw JSONRPCError(RPC_DATABASE_ERROR, "Error: Ban database not loaded"); } @@ -667,7 +667,7 @@ static UniValue setnetworkactive(const JSONRPCRequest& request) RPCHelpMan{"setnetworkactive", "\nDisable/enable all p2p network activity.\n", { - {"state", RPCArg::Type::BOOL, /* opt */ false, /* default_val */ "", "true to enable networking, false to disable"}, + {"state", RPCArg::Type::BOOL, RPCArg::Optional::NO, "true to enable networking, false to disable"}, }, RPCResults{}, RPCExamples{""}, @@ -691,7 +691,7 @@ static UniValue getnodeaddresses(const JSONRPCRequest& request) RPCHelpMan{"getnodeaddresses", "\nReturn known addresses which can potentially be used to find new nodes in the network\n", { - {"count", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1", "How many addresses to return. Limited to the smaller of " + std::to_string(ADDRMAN_GETADDR_MAX) + " or " + std::to_string(ADDRMAN_GETADDR_MAX_PCT) + "% of all known addresses."}, + {"count", RPCArg::Type::NUM, /* default */ "1", "How many addresses to return. Limited to the smaller of " + std::to_string(ADDRMAN_GETADDR_MAX) + " or " + std::to_string(ADDRMAN_GETADDR_MAX_PCT) + "% of all known addresses."}, }, RPCResult{ "[\n" diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index b3b9d8af09..d19afaa8a1 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -6,6 +6,7 @@ #include <chain.h> #include <coins.h> #include <compat/byteswap.h> +#include <consensus/tx_verify.h> #include <consensus/validation.h> #include <core_io.h> #include <index/txindex.h> @@ -13,10 +14,11 @@ #include <key_io.h> #include <keystore.h> #include <merkleblock.h> -#include <net.h> +#include <node/transaction.h> #include <policy/policy.h> #include <policy/rbf.h> #include <primitives/transaction.h> +#include <psbt.h> #include <rpc/rawtransaction.h> #include <rpc/server.h> #include <rpc/util.h> @@ -24,13 +26,14 @@ #include <script/script_error.h> #include <script/sign.h> #include <script/standard.h> -#include <txmempool.h> #include <uint256.h> +#include <util/bip32.h> #include <util/strencodings.h> #include <validation.h> #include <validationinterface.h> -#include <future> + +#include <numeric> #include <stdint.h> #include <univalue.h> @@ -64,9 +67,7 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& static UniValue getrawtransaction(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) - throw std::runtime_error( - RPCHelpMan{ + const RPCHelpMan help{ "getrawtransaction", "\nReturn the raw transaction data.\n" @@ -76,15 +77,14 @@ static UniValue getrawtransaction(const JSONRPCRequest& request) "will return the transaction if it is in the mempool, or if -txindex is enabled and the transaction\n" "is in a block in the blockchain.\n" - "\nHint: use getmempoolentry to fetch a specific transaction from the mempool.\n" - "Or use gettransaction for wallet transactions.\n" + "\nHint: Use gettransaction for wallet transactions.\n" "\nIf verbose is 'true', returns an Object with information about 'txid'.\n" "If verbose is 'false' or omitted, returns a string that is serialized, hex-encoded data for 'txid'.\n", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id"}, - {"verbose", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "If false, return a string, otherwise return a json object"}, - {"blockhash", RPCArg::Type::STR_HEX, /* opt */ true, /* default_val */ "null", "The block in which to look for the transaction"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"verbose", RPCArg::Type::BOOL, /* default */ "false", "If false, return a string, otherwise return a json object"}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED_NAMED_ARG, "The block in which to look for the transaction"}, }, { RPCResult{"if verbose is not set or set to false", @@ -145,7 +145,11 @@ static UniValue getrawtransaction(const JSONRPCRequest& request) + HelpExampleCli("getrawtransaction", "\"mytxid\" false \"myblockhash\"") + HelpExampleCli("getrawtransaction", "\"mytxid\" true \"myblockhash\"") }, - }.ToString()); + }; + + if (request.fHelp || !help.IsValidNumArgs(request.params.size())) { + throw std::runtime_error(help.ToString()); + } bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); @@ -218,12 +222,12 @@ static UniValue gettxoutproof(const JSONRPCRequest& request) "you need to maintain a transaction index, using the -txindex command line option or\n" "specify the block in which the transaction is included manually (by blockhash).\n", { - {"txids", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "A json array of txids to filter", + {"txids", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of txids to filter", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "A transaction hash"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A transaction hash"}, }, }, - {"blockhash", RPCArg::Type::STR_HEX, /* opt */ true, /* default_val */ "null", "If specified, looks for txid in the block with this hash"}, + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED_NAMED_ARG, "If specified, looks for txid in the block with this hash"}, }, RPCResult{ "\"data\" (string) A string that is a serialized, hex-encoded data for the proof.\n" @@ -311,7 +315,7 @@ static UniValue verifytxoutproof(const JSONRPCRequest& request) "\nVerifies that a proof points to a transaction in a block, returning the transaction it commits to\n" "and throwing an RPC error if the block is not in our best chain\n", { - {"proof", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The hex-encoded proof generated by gettxoutproof"}, + {"proof", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex-encoded proof generated by gettxoutproof"}, }, RPCResult{ "[\"txid\"] (array, strings) The txid(s) which the proof commits to, or empty array if the proof can not be validated.\n" @@ -472,36 +476,36 @@ static UniValue createrawtransaction(const JSONRPCRequest& request) "Note that the transaction's inputs are not signed, and\n" "it is not stored in the wallet or transmitted to the network.\n", { - {"inputs", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "A json array of json objects", + {"inputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of json objects", { - {"", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id"}, - {"vout", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The output number"}, - {"sequence", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "depends on the value of the 'replaceable' and 'locktime' arguments", "The sequence number"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, + {"sequence", RPCArg::Type::NUM, /* default */ "depends on the value of the 'replaceable' and 'locktime' arguments", "The sequence number"}, }, }, }, }, - {"outputs", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "a json array with outputs (key-value pairs), where none of the keys are duplicated.\n" + {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "a json array with outputs (key-value pairs), where none of the keys are duplicated.\n" "That is, each address can only appear once and there can only be one 'data' object.\n" "For compatibility reasons, a dictionary, which holds the key-value pairs directly, is also\n" " accepted as second parameter.", { - {"", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"address", RPCArg::Type::AMOUNT, /* opt */ false, /* default_val */ "", "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT}, + {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT}, }, }, - {"", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"data", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "A key-value pair. The key must be \"data\", the value is hex-encoded data"}, + {"data", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "A key-value pair. The key must be \"data\", the value is hex-encoded data"}, }, }, }, }, - {"locktime", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "Raw locktime. Non-0 value also locktime-activates inputs"}, - {"replaceable", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Marks this transaction as BIP125-replaceable.\n" + {"locktime", RPCArg::Type::NUM, /* default */ "0", "Raw locktime. Non-0 value also locktime-activates inputs"}, + {"replaceable", RPCArg::Type::BOOL, /* default */ "false", "Marks this transaction as BIP125-replaceable.\n" " Allows this transaction to be replaced by a transaction with higher fees. If provided, it is an error if explicit sequence numbers are incompatible."}, }, RPCResult{ @@ -536,8 +540,8 @@ static UniValue decoderawtransaction(const JSONRPCRequest& request) RPCHelpMan{"decoderawtransaction", "\nReturn a JSON object representing the serialized, hex-encoded transaction.\n", { - {"hexstring", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction hex string"}, - {"iswitness", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "depends on heuristic tests", "Whether the transaction hex is a serialized witness transaction\n" + {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction hex string"}, + {"iswitness", RPCArg::Type::BOOL, /* default */ "depends on heuristic tests", "Whether the transaction hex is a serialized witness transaction\n" " If iswitness is not present, heuristic tests will be used in decoding"}, }, RPCResult{ @@ -611,7 +615,7 @@ static UniValue decodescript(const JSONRPCRequest& request) RPCHelpMan{"decodescript", "\nDecode a hex-encoded script.\n", { - {"hexstring", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "the hex-encoded script"}, + {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hex-encoded script"}, }, RPCResult{ "{\n" @@ -711,9 +715,9 @@ static UniValue combinerawtransaction(const JSONRPCRequest& request) "The combined transaction may be another partially signed transaction or a \n" "fully signed transaction.", { - {"txs", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "A json array of hex strings of partially signed transactions", + {"txs", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of hex strings of partially signed transactions", { - {"hexstring", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "A transaction hash"}, + {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A transaction hash"}, }, }, }, @@ -856,15 +860,25 @@ UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, con RPCTypeCheckObj(prevOut, { {"redeemScript", UniValueType(UniValue::VSTR)}, - }); - UniValue v = find_value(prevOut, "redeemScript"); - if (!v.isNull()) { - std::vector<unsigned char> rsData(ParseHexV(v, "redeemScript")); + {"witnessScript", UniValueType(UniValue::VSTR)}, + }, true); + UniValue rs = find_value(prevOut, "redeemScript"); + if (!rs.isNull()) { + std::vector<unsigned char> rsData(ParseHexV(rs, "redeemScript")); CScript redeemScript(rsData.begin(), rsData.end()); keystore->AddCScript(redeemScript); // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). + // This is only for compatibility, it is encouraged to use the explicit witnessScript field instead. keystore->AddCScript(GetScriptForWitness(redeemScript)); } + UniValue ws = find_value(prevOut, "witnessScript"); + if (!ws.isNull()) { + std::vector<unsigned char> wsData(ParseHexV(ws, "witnessScript")); + CScript witnessScript(wsData.begin(), wsData.end()); + keystore->AddCScript(witnessScript); + // Automatically also add the P2WSH wrapped version of the script (to deal with P2SH-P2WSH). + keystore->AddCScript(GetScriptForWitness(witnessScript)); + } } } } @@ -936,26 +950,27 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) "The third optional argument (may be null) is an array of previous transaction outputs that\n" "this transaction depends on but may not yet be in the block chain.\n", { - {"hexstring", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The transaction hex string"}, - {"privkeys", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "A json array of base58-encoded private keys for signing", + {"hexstring", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction hex string"}, + {"privkeys", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of base58-encoded private keys for signing", { - {"privatekey", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "private key in base58-encoding"}, + {"privatekey", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "private key in base58-encoding"}, }, }, - {"prevtxs", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "null", "A json array of previous dependent transaction outputs", + {"prevtxs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "A json array of previous dependent transaction outputs", { - {"", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id"}, - {"vout", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The output number"}, - {"scriptPubKey", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "script key"}, - {"redeemScript", RPCArg::Type::STR_HEX, /* opt */ true, /* default_val */ "omitted", "(required for P2SH or P2WSH) redeem script"}, - {"amount", RPCArg::Type::AMOUNT, /* opt */ false, /* default_val */ "", "The amount spent"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, + {"scriptPubKey", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "script key"}, + {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH) redeem script"}, + {"witnessScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2WSH or P2SH-P2WSH) witness script"}, + {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount spent"}, }, }, }, }, - {"sighashtype", RPCArg::Type::STR, /* opt */ true, /* default_val */ "ALL", "The signature hash type. Must be one of:\n" + {"sighashtype", RPCArg::Type::STR, /* default */ "ALL", "The signature hash type. Must be one of:\n" " \"ALL\"\n" " \"NONE\"\n" " \"SINGLE\"\n" @@ -1023,8 +1038,8 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) "\nSubmits raw transaction (serialized, hex-encoded) to local node and network.\n" "\nAlso see createrawtransaction and signrawtransactionwithkey calls.\n", { - {"hexstring", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The hex string of the raw transaction"}, - {"allowhighfees", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Allow high fees"}, + {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, + {"allowhighfees", RPCArg::Type::BOOL, /* default */ "false", "Allow high fees"}, }, RPCResult{ "\"hex\" (string) The transaction hash in hex\n" @@ -1041,8 +1056,6 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) }, }.ToString()); - std::promise<void> promise; - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL}); // parse hex string from parameter @@ -1050,67 +1063,18 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) if (!DecodeHexTx(mtx, request.params[0].get_str())) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - const uint256& hashTx = tx->GetHash(); - CAmount nMaxRawTxFee = maxTxFee; - if (!request.params[1].isNull() && request.params[1].get_bool()) - nMaxRawTxFee = 0; - - { // cs_main scope - LOCK(cs_main); - CCoinsViewCache &view = *pcoinsTip; - bool fHaveChain = false; - for (size_t o = 0; !fHaveChain && o < tx->vout.size(); o++) { - const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); - fHaveChain = !existingCoin.IsSpent(); - } - bool fHaveMempool = mempool.exists(hashTx); - if (!fHaveMempool && !fHaveChain) { - // push to local node and sync with wallets - CValidationState state; - bool fMissingInputs; - if (!AcceptToMemoryPool(mempool, state, std::move(tx), &fMissingInputs, - nullptr /* plTxnReplaced */, false /* bypass_limits */, nMaxRawTxFee)) { - if (state.IsInvalid()) { - throw JSONRPCError(RPC_TRANSACTION_REJECTED, FormatStateMessage(state)); - } else { - if (fMissingInputs) { - throw JSONRPCError(RPC_TRANSACTION_ERROR, "Missing inputs"); - } - throw JSONRPCError(RPC_TRANSACTION_ERROR, FormatStateMessage(state)); - } - } else { - // If wallet is enabled, ensure that the wallet has been made aware - // of the new transaction prior to returning. This prevents a race - // where a user might call sendrawtransaction with a transaction - // to/from their wallet, immediately call some wallet RPC, and get - // a stale result because callbacks have not yet been processed. - CallFunctionInValidationInterfaceQueue([&promise] { - promise.set_value(); - }); - } - } else if (fHaveChain) { - throw JSONRPCError(RPC_TRANSACTION_ALREADY_IN_CHAIN, "transaction already in block chain"); - } else { - // Make sure we don't block forever if re-sending - // a transaction already in mempool. - promise.set_value(); + bool allowhighfees = false; + if (!request.params[1].isNull()) allowhighfees = request.params[1].get_bool(); + const CAmount highfee{allowhighfees ? 0 : ::maxTxFee}; + uint256 txid; + std::string err_string; + const TransactionError err = BroadcastTransaction(tx, txid, err_string, highfee); + if (TransactionError::OK != err) { + throw JSONRPCTransactionError(err, err_string); } - } // cs_main - - promise.get_future().wait(); - - if(!g_connman) - throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); - - CInv inv(MSG_TX, hashTx); - g_connman->ForEachNode([&inv](CNode* pnode) - { - pnode->PushInventory(inv); - }); - - return hashTx.GetHex(); + return txid.GetHex(); } static UniValue testmempoolaccept(const JSONRPCRequest& request) @@ -1122,13 +1086,13 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) "\nThis checks if the transaction violates the consensus or policy rules.\n" "\nSee sendrawtransaction call.\n", { - {"rawtxs", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "An array of hex strings of raw transactions.\n" + {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings of raw transactions.\n" " Length must be one for now.", { - {"rawtx", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", ""}, + {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, }, - {"allowhighfees", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Allow high fees"}, + {"allowhighfees", RPCArg::Type::BOOL, /* default */ "false", "Allow high fees"}, }, RPCResult{ "[ (array) The result of the mempool acceptance test for each raw transaction in the input array.\n" @@ -1223,7 +1187,7 @@ UniValue decodepsbt(const JSONRPCRequest& request) RPCHelpMan{"decodepsbt", "\nReturn a JSON object representing the serialized, base64-encoded partially signed Bitcoin transaction.\n", { - {"psbt", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The PSBT base64 string"}, + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "The PSBT base64 string"}, }, RPCResult{ "{\n" @@ -1323,7 +1287,7 @@ UniValue decodepsbt(const JSONRPCRequest& request) // Unserialize the transactions PartiallySignedTransaction psbtx; std::string error; - if (!DecodePSBT(psbtx, request.params[0].get_str(), error)) { + if (!DecodeBase64PSBT(psbtx, request.params[0].get_str(), error)) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); } @@ -1499,9 +1463,9 @@ UniValue combinepsbt(const JSONRPCRequest& request) "\nCombine multiple partially signed Bitcoin transactions into one transaction.\n" "Implements the Combiner role.\n", { - {"txs", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "A json array of base64 strings of partially signed transactions", + {"txs", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of base64 strings of partially signed transactions", { - {"psbt", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "A base64 string of a PSBT"}, + {"psbt", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A base64 string of a PSBT"}, }, }, }, @@ -1518,26 +1482,22 @@ UniValue combinepsbt(const JSONRPCRequest& request) // Unserialize the transactions std::vector<PartiallySignedTransaction> psbtxs; UniValue txs = request.params[0].get_array(); + if (txs.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter 'txs' cannot be empty"); + } for (unsigned int i = 0; i < txs.size(); ++i) { PartiallySignedTransaction psbtx; std::string error; - if (!DecodePSBT(psbtx, txs[i].get_str(), error)) { + if (!DecodeBase64PSBT(psbtx, txs[i].get_str(), error)) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); } psbtxs.push_back(psbtx); } - PartiallySignedTransaction merged_psbt(psbtxs[0]); // Copy the first one - - // Merge - for (auto it = std::next(psbtxs.begin()); it != psbtxs.end(); ++it) { - if (*it != merged_psbt) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "PSBTs do not refer to the same transactions."); - } - merged_psbt.Merge(*it); - } - if (!merged_psbt.IsSane()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Merged PSBT is inconsistent"); + PartiallySignedTransaction merged_psbt; + const TransactionError error = CombinePSBTs(merged_psbt, psbtxs); + if (error != TransactionError::OK) { + throw JSONRPCTransactionError(error); } UniValue result(UniValue::VOBJ); @@ -1556,8 +1516,8 @@ UniValue finalizepsbt(const JSONRPCRequest& request) "created which has the final_scriptSig and final_scriptWitness fields filled for inputs that are complete.\n" "Implements the Finalizer and Extractor roles.\n", { - {"psbt", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "A base64 string of a PSBT"}, - {"extract", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "If true and the transaction is complete,\n" + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "A base64 string of a PSBT"}, + {"extract", RPCArg::Type::BOOL, /* default */ "true", "If true and the transaction is complete,\n" " extract and return the complete transaction in normal network serialization instead of the PSBT."}, }, RPCResult{ @@ -1578,33 +1538,27 @@ UniValue finalizepsbt(const JSONRPCRequest& request) // Unserialize the transactions PartiallySignedTransaction psbtx; std::string error; - if (!DecodePSBT(psbtx, request.params[0].get_str(), error)) { + if (!DecodeBase64PSBT(psbtx, request.params[0].get_str(), error)) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); } - // Finalize input signatures -- in case we have partial signatures that add up to a complete - // signature, but have not combined them yet (e.g. because the combiner that created this - // PartiallySignedTransaction did not understand them), this will combine them into a final - // script. - bool complete = true; - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - complete &= SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, SIGHASH_ALL); - } + bool extract = request.params[1].isNull() || (!request.params[1].isNull() && request.params[1].get_bool()); + + CMutableTransaction mtx; + bool complete = FinalizeAndExtractPSBT(psbtx, mtx); UniValue result(UniValue::VOBJ); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - bool extract = request.params[1].isNull() || (!request.params[1].isNull() && request.params[1].get_bool()); + std::string result_str; + if (complete && extract) { - CMutableTransaction mtx(*psbtx.tx); - for (unsigned int i = 0; i < mtx.vin.size(); ++i) { - mtx.vin[i].scriptSig = psbtx.inputs[i].final_script_sig; - mtx.vin[i].scriptWitness = psbtx.inputs[i].final_script_witness; - } ssTx << mtx; - result.pushKV("hex", HexStr(ssTx.str())); + result_str = HexStr(ssTx.str()); + result.pushKV("hex", result_str); } else { ssTx << psbtx; - result.pushKV("psbt", EncodeBase64(ssTx.str())); + result_str = EncodeBase64(ssTx.str()); + result.pushKV("psbt", result_str); } result.pushKV("complete", complete); @@ -1619,36 +1573,36 @@ UniValue createpsbt(const JSONRPCRequest& request) "\nCreates a transaction in the Partially Signed Transaction format.\n" "Implements the Creator role.\n", { - {"inputs", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "A json array of json objects", + {"inputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of json objects", { - {"", RPCArg::Type::OBJ, /* opt */ false, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id"}, - {"vout", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The output number"}, - {"sequence", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "depends on the value of the 'replaceable' and 'locktime' arguments", "The sequence number"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, + {"sequence", RPCArg::Type::NUM, /* default */ "depends on the value of the 'replaceable' and 'locktime' arguments", "The sequence number"}, }, }, }, }, - {"outputs", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "a json array with outputs (key-value pairs), where none of the keys are duplicated.\n" + {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "a json array with outputs (key-value pairs), where none of the keys are duplicated.\n" "That is, each address can only appear once and there can only be one 'data' object.\n" "For compatibility reasons, a dictionary, which holds the key-value pairs directly, is also\n" " accepted as second parameter.", { - {"", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"address", RPCArg::Type::AMOUNT, /* opt */ false, /* default_val */ "", "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT}, + {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT}, }, }, - {"", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"data", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "A key-value pair. The key must be \"data\", the value is hex-encoded data"}, + {"data", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "A key-value pair. The key must be \"data\", the value is hex-encoded data"}, }, }, }, }, - {"locktime", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "Raw locktime. Non-0 value also locktime-activates inputs"}, - {"replaceable", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Marks this transaction as BIP125 replaceable.\n" + {"locktime", RPCArg::Type::NUM, /* default */ "0", "Raw locktime. Non-0 value also locktime-activates inputs"}, + {"replaceable", RPCArg::Type::BOOL, /* default */ "false", "Marks this transaction as BIP125 replaceable.\n" " Allows this transaction to be replaced by a transaction with higher fees. If provided, it is an error if explicit sequence numbers are incompatible."}, }, RPCResult{ @@ -1695,10 +1649,10 @@ UniValue converttopsbt(const JSONRPCRequest& request) "\nConverts a network serialized transaction to a PSBT. This should be used only with createrawtransaction and fundrawtransaction\n" "createpsbt and walletcreatefundedpsbt should be used for new applications.\n", { - {"hexstring", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The hex string of a raw transaction"}, - {"permitsigdata", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "If true, any signatures in the input will be discarded and conversion.\n" + {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of a raw transaction"}, + {"permitsigdata", RPCArg::Type::BOOL, /* default */ "false", "If true, any signatures in the input will be discarded and conversion.\n" " will continue. If false, RPC will fail if any signatures are present."}, - {"iswitness", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "depends on heuristic tests", "Whether the transaction hex is a serialized witness transaction.\n" + {"iswitness", RPCArg::Type::BOOL, /* default */ "depends on heuristic tests", "Whether the transaction hex is a serialized witness transaction.\n" " If iswitness is not present, heuristic tests will be used in decoding. If true, only witness deserializaion\n" " will be tried. If false, only non-witness deserialization will be tried. Only has an effect if\n" " permitsigdata is true."}, @@ -1754,6 +1708,338 @@ UniValue converttopsbt(const JSONRPCRequest& request) return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); } +UniValue utxoupdatepsbt(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) { + throw std::runtime_error( + RPCHelpMan{"utxoupdatepsbt", + "\nUpdates a PSBT with witness UTXOs retrieved from the UTXO set or the mempool.\n", + { + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "A base64 string of a PSBT"} + }, + RPCResult { + " \"psbt\" (string) The base64-encoded partially signed transaction with inputs updated\n" + }, + RPCExamples { + HelpExampleCli("utxoupdatepsbt", "\"psbt\"") + }}.ToString()); + } + + RPCTypeCheck(request.params, {UniValue::VSTR}, true); + + // Unserialize the transactions + PartiallySignedTransaction psbtx; + std::string error; + if (!DecodeBase64PSBT(psbtx, request.params[0].get_str(), error)) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); + } + + // Fetch previous transactions (inputs): + CCoinsView viewDummy; + CCoinsViewCache view(&viewDummy); + { + LOCK2(cs_main, mempool.cs); + CCoinsViewCache &viewChain = *pcoinsTip; + CCoinsViewMemPool viewMempool(&viewChain, mempool); + view.SetBackend(viewMempool); // temporarily switch cache backend to db+mempool view + + for (const CTxIn& txin : psbtx.tx->vin) { + view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail. + } + + view.SetBackend(viewDummy); // switch back to avoid locking mempool for too long + } + + // Fill the inputs + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs.at(i); + + if (input.non_witness_utxo || !input.witness_utxo.IsNull()) { + continue; + } + + const Coin& coin = view.AccessCoin(psbtx.tx->vin[i].prevout); + + std::vector<std::vector<unsigned char>> solutions_data; + txnouttype which_type = Solver(coin.out.scriptPubKey, solutions_data); + if (which_type == TX_WITNESS_V0_SCRIPTHASH || which_type == TX_WITNESS_V0_KEYHASH || which_type == TX_WITNESS_UNKNOWN) { + input.witness_utxo = coin.out; + } + } + + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); +} + +UniValue joinpsbts(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) { + throw std::runtime_error( + RPCHelpMan{"joinpsbts", + "\nJoins multiple distinct PSBTs with different inputs and outputs into one PSBT with inputs and outputs from all of the PSBTs\n" + "No input in any of the PSBTs can be in more than one of the PSBTs.\n", + { + {"txs", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of base64 strings of partially signed transactions", + { + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "A base64 string of a PSBT"} + }} + }, + RPCResult { + " \"psbt\" (string) The base64-encoded partially signed transaction\n" + }, + RPCExamples { + HelpExampleCli("joinpsbts", "\"psbt\"") + }}.ToString()); + } + + RPCTypeCheck(request.params, {UniValue::VARR}, true); + + // Unserialize the transactions + std::vector<PartiallySignedTransaction> psbtxs; + UniValue txs = request.params[0].get_array(); + + if (txs.size() <= 1) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "At least two PSBTs are required to join PSBTs."); + } + + int32_t best_version = 1; + uint32_t best_locktime = 0xffffffff; + for (unsigned int i = 0; i < txs.size(); ++i) { + PartiallySignedTransaction psbtx; + std::string error; + if (!DecodeBase64PSBT(psbtx, txs[i].get_str(), error)) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); + } + psbtxs.push_back(psbtx); + // Choose the highest version number + if (psbtx.tx->nVersion > best_version) { + best_version = psbtx.tx->nVersion; + } + // Choose the lowest lock time + if (psbtx.tx->nLockTime < best_locktime) { + best_locktime = psbtx.tx->nLockTime; + } + } + + // Create a blank psbt where everything will be added + PartiallySignedTransaction merged_psbt; + merged_psbt.tx = CMutableTransaction(); + merged_psbt.tx->nVersion = best_version; + merged_psbt.tx->nLockTime = best_locktime; + + // Merge + for (auto& psbt : psbtxs) { + for (unsigned int i = 0; i < psbt.tx->vin.size(); ++i) { + if (!merged_psbt.AddInput(psbt.tx->vin[i], psbt.inputs[i])) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input %s:%d exists in multiple PSBTs", psbt.tx->vin[i].prevout.hash.ToString().c_str(), psbt.tx->vin[i].prevout.n)); + } + } + for (unsigned int i = 0; i < psbt.tx->vout.size(); ++i) { + merged_psbt.AddOutput(psbt.tx->vout[i], psbt.outputs[i]); + } + merged_psbt.unknown.insert(psbt.unknown.begin(), psbt.unknown.end()); + } + + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << merged_psbt; + return EncodeBase64((unsigned char*)ssTx.data(), ssTx.size()); +} + +UniValue analyzepsbt(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() != 1) { + throw std::runtime_error( + RPCHelpMan{"analyzepsbt", + "\nAnalyzes and provides information about the current status of a PSBT and its inputs\n", + { + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "A base64 string of a PSBT"} + }, + RPCResult { + "{\n" + " \"inputs\" : [ (array of json objects)\n" + " {\n" + " \"has_utxo\" : true|false (boolean) Whether a UTXO is provided\n" + " \"is_final\" : true|false (boolean) Whether the input is finalized\n" + " \"missing\" : { (json object, optional) Things that are missing that are required to complete this input\n" + " \"pubkeys\" : [ (array)\n" + " \"keyid\" (string) Public key ID, hash160 of the public key, of a public key whose BIP 32 derivation path is missing\n" + " ]\n" + " \"signatures\" : [ (array)\n" + " \"keyid\" (string) Public key ID, hash160 of the public key, of a public key whose signature is missing\n" + " ]\n" + " \"redeemscript\" : \"hash\" (string) Hash160 of the redeemScript that is missing\n" + " \"witnessscript\" : \"hash\" (string) SHA256 of the witnessScript that is missing\n" + " }\n" + " \"next\" : \"role\" (string) Role of the next person that this input needs to go to\n" + " }\n" + " ,...\n" + " ]\n" + " \"estimated_vsize\" : vsize (numeric) Estimated vsize of the final signed transaction\n" + " \"estimated_feerate\" : feerate (numeric, optional) Estimated feerate of the final signed transaction. Shown only if all UTXO slots in the PSBT have been filled.\n" + " \"fee\" : fee (numeric, optional) The transaction fee paid. Shown only if all UTXO slots in the PSBT have been filled.\n" + " \"next\" : \"role\" (string) Role of the next person that this psbt needs to go to\n" + "}\n" + }, + RPCExamples { + HelpExampleCli("analyzepsbt", "\"psbt\"") + }}.ToString()); + } + + RPCTypeCheck(request.params, {UniValue::VSTR}); + + // Unserialize the transaction + PartiallySignedTransaction psbtx; + std::string error; + if (!DecodeBase64PSBT(psbtx, request.params[0].get_str(), error)) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); + } + + // Go through each input and build status + UniValue result(UniValue::VOBJ); + UniValue inputs_result(UniValue::VARR); + bool calc_fee = true; + bool all_final = true; + bool only_missing_sigs = true; + bool only_missing_final = false; + CAmount in_amt = 0; + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs[i]; + UniValue input_univ(UniValue::VOBJ); + UniValue missing(UniValue::VOBJ); + + // Check for a UTXO + CTxOut utxo; + if (psbtx.GetInputUTXO(utxo, i)) { + in_amt += utxo.nValue; + input_univ.pushKV("has_utxo", true); + } else { + input_univ.pushKV("has_utxo", false); + input_univ.pushKV("is_final", false); + input_univ.pushKV("next", "updater"); + calc_fee = false; + } + + // Check if it is final + if (!utxo.IsNull() && !PSBTInputSigned(input)) { + input_univ.pushKV("is_final", false); + all_final = false; + + // Figure out what is missing + SignatureData outdata; + bool complete = SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, &outdata); + + // Things are missing + if (!complete) { + if (!outdata.missing_pubkeys.empty()) { + // Missing pubkeys + UniValue missing_pubkeys_univ(UniValue::VARR); + for (const CKeyID& pubkey : outdata.missing_pubkeys) { + missing_pubkeys_univ.push_back(HexStr(pubkey)); + } + missing.pushKV("pubkeys", missing_pubkeys_univ); + } + if (!outdata.missing_redeem_script.IsNull()) { + // Missing redeemScript + missing.pushKV("redeemscript", HexStr(outdata.missing_redeem_script)); + } + if (!outdata.missing_witness_script.IsNull()) { + // Missing witnessScript + missing.pushKV("witnessscript", HexStr(outdata.missing_witness_script)); + } + if (!outdata.missing_sigs.empty()) { + // Missing sigs + UniValue missing_sigs_univ(UniValue::VARR); + for (const CKeyID& pubkey : outdata.missing_sigs) { + missing_sigs_univ.push_back(HexStr(pubkey)); + } + missing.pushKV("signatures", missing_sigs_univ); + } + input_univ.pushKV("missing", missing); + + // If we are only missing signatures and nothing else, then next is signer + if (outdata.missing_pubkeys.empty() && outdata.missing_redeem_script.IsNull() && outdata.missing_witness_script.IsNull() && !outdata.missing_sigs.empty()) { + input_univ.pushKV("next", "signer"); + } else { + only_missing_sigs = false; + input_univ.pushKV("next", "updater"); + } + } else { + only_missing_final = true; + input_univ.pushKV("next", "finalizer"); + } + } else if (!utxo.IsNull()){ + input_univ.pushKV("is_final", true); + } + inputs_result.push_back(input_univ); + } + result.pushKV("inputs", inputs_result); + + if (all_final) { + only_missing_sigs = false; + result.pushKV("next", "extractor"); + } + if (calc_fee) { + // Get the output amount + CAmount out_amt = std::accumulate(psbtx.tx->vout.begin(), psbtx.tx->vout.end(), 0, + [](int a, const CTxOut& b) { + return a += b.nValue; + } + ); + + // Get the fee + CAmount fee = in_amt - out_amt; + + // Estimate the size + CMutableTransaction mtx(*psbtx.tx); + CCoinsView view_dummy; + CCoinsViewCache view(&view_dummy); + bool success = true; + + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + PSBTInput& input = psbtx.inputs[i]; + if (SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, 1, nullptr, true)) { + mtx.vin[i].scriptSig = input.final_script_sig; + mtx.vin[i].scriptWitness = input.final_script_witness; + + Coin newcoin; + if (!psbtx.GetInputUTXO(newcoin.out, i)) { + success = false; + break; + } + newcoin.nHeight = 1; + view.AddCoin(psbtx.tx->vin[i].prevout, std::move(newcoin), true); + } else { + success = false; + break; + } + } + + if (success) { + CTransaction ctx = CTransaction(mtx); + size_t size = GetVirtualTransactionSize(ctx, GetTransactionSigOpCost(ctx, view, STANDARD_SCRIPT_VERIFY_FLAGS)); + result.pushKV("estimated_vsize", (int)size); + // Estimate fee rate + CFeeRate feerate(fee, size); + result.pushKV("estimated_feerate", feerate.ToString()); + } + result.pushKV("fee", ValueFromAmount(fee)); + + if (only_missing_sigs) { + result.pushKV("next", "signer"); + } else if (only_missing_final) { + result.pushKV("next", "finalizer"); + } else if (all_final) { + result.pushKV("next", "extractor"); + } else { + result.pushKV("next", "updater"); + } + } else { + result.pushKV("next", "updater"); + } + return result; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -1772,6 +2058,9 @@ static const CRPCCommand commands[] = { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, { "rawtransactions", "createpsbt", &createpsbt, {"inputs","outputs","locktime","replaceable"} }, { "rawtransactions", "converttopsbt", &converttopsbt, {"hexstring","permitsigdata","iswitness"} }, + { "rawtransactions", "utxoupdatepsbt", &utxoupdatepsbt, {"psbt"} }, + { "rawtransactions", "joinpsbts", &joinpsbts, {"txs"} }, + { "rawtransactions", "analyzepsbt", &analyzepsbt, {"psbt"} }, { "blockchain", "gettxoutproof", &gettxoutproof, {"txids", "blockhash"} }, { "blockchain", "verifytxoutproof", &verifytxoutproof, {"proof"} }, diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 2ed74547b9..cd90573da0 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -230,7 +230,7 @@ UniValue help(const JSONRPCRequest& jsonRequest) RPCHelpMan{"help", "\nList all commands, or get help for a specified command.\n", { - {"command", RPCArg::Type::STR, /* opt */ true, /* default_val */ "all commands", "The command to get help on"}, + {"command", RPCArg::Type::STR, /* default */ "all commands", "The command to get help on"}, }, RPCResult{ "\"text\" (string) The help text\n" diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 4275cc09a8..10979b43b0 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -1,10 +1,9 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key_io.h> #include <keystore.h> -#include <rpc/protocol.h> #include <rpc/util.h> #include <tinyformat.h> #include <util/strencodings.h> @@ -129,6 +128,43 @@ UniValue DescribeAddress(const CTxDestination& dest) return boost::apply_visitor(DescribeAddressVisitor(), dest); } +unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target) +{ + int target = value.get_int(); + if (target < 1 || (unsigned int)target > max_target) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid conf_target, must be between %u - %u", 1, max_target)); + } + return (unsigned int)target; +} + +RPCErrorCode RPCErrorFromTransactionError(TransactionError terr) +{ + switch (terr) { + case TransactionError::MEMPOOL_REJECTED: + return RPC_TRANSACTION_REJECTED; + case TransactionError::ALREADY_IN_CHAIN: + return RPC_TRANSACTION_ALREADY_IN_CHAIN; + case TransactionError::P2P_DISABLED: + return RPC_CLIENT_P2P_DISABLED; + case TransactionError::INVALID_PSBT: + case TransactionError::PSBT_MISMATCH: + return RPC_INVALID_PARAMETER; + case TransactionError::SIGHASH_MISMATCH: + return RPC_DESERIALIZATION_ERROR; + default: break; + } + return RPC_TRANSACTION_ERROR; +} + +UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_string) +{ + if (err_string.length() > 0) { + return JSONRPCError(RPCErrorFromTransactionError(terr), err_string); + } else { + return JSONRPCError(RPCErrorFromTransactionError(terr), TransactionErrorString(terr)); + } +} + struct Section { Section(const std::string& left, const std::string& right) : m_left{left}, m_right{right} {} @@ -161,6 +197,7 @@ struct Sections { case RPCArg::Type::STR: case RPCArg::Type::NUM: case RPCArg::Type::AMOUNT: + case RPCArg::Type::RANGE: case RPCArg::Type::BOOL: { if (outer_type == OuterType::NAMED_ARG) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; @@ -170,12 +207,12 @@ struct Sections { left += outer_type == OuterType::OBJ ? arg.ToStringObj(/* oneline */ false) : arg.ToString(/* oneline */ false); } left += ","; - PushSection({left, arg.ToDescriptionString(/* implicitly_required */ outer_type == OuterType::ARR)}); + PushSection({left, arg.ToDescriptionString()}); break; } case RPCArg::Type::OBJ: case RPCArg::Type::OBJ_USER_KEYS: { - const auto right = outer_type == OuterType::NAMED_ARG ? "" : arg.ToDescriptionString(/* implicitly_required */ outer_type == OuterType::ARR); + const auto right = outer_type == OuterType::NAMED_ARG ? "" : arg.ToDescriptionString(); PushSection({indent + "{", right}); for (const auto& arg_inner : arg.m_inner) { Push(arg_inner, current_indent + 2, OuterType::OBJ); @@ -190,7 +227,7 @@ struct Sections { auto left = indent; left += outer_type == OuterType::OBJ ? "\"" + arg.m_name + "\": " : ""; left += "["; - const auto right = outer_type == OuterType::NAMED_ARG ? "" : arg.ToDescriptionString(/* implicitly_required */ outer_type == OuterType::ARR); + const auto right = outer_type == OuterType::NAMED_ARG ? "" : arg.ToDescriptionString(); PushSection({left, right}); for (const auto& arg_inner : arg.m_inner) { Push(arg_inner, current_indent + 2, OuterType::ARR); @@ -275,6 +312,17 @@ std::string RPCExamples::ToDescriptionString() const return m_examples.empty() ? m_examples : "\nExamples:\n" + m_examples; } +bool RPCHelpMan::IsValidNumArgs(size_t num_args) const +{ + size_t num_required_args = 0; + for (size_t n = m_args.size(); n > 0; --n) { + if (!m_args.at(n - 1).IsOptional()) { + num_required_args = n; + break; + } + } + return num_required_args <= num_args && num_args <= m_args.size(); +} std::string RPCHelpMan::ToString() const { std::string ret; @@ -283,8 +331,9 @@ std::string RPCHelpMan::ToString() const ret += m_name; bool was_optional{false}; for (const auto& arg : m_args) { + const bool optional = arg.IsOptional(); ret += " "; - if (arg.m_optional) { + if (optional) { if (!was_optional) ret += "( "; was_optional = true; } else { @@ -324,7 +373,16 @@ std::string RPCHelpMan::ToString() const return ret; } -std::string RPCArg::ToDescriptionString(const bool implicitly_required) const +bool RPCArg::IsOptional() const +{ + if (m_fallback.which() == 1) { + return true; + } else { + return RPCArg::Optional::NO != boost::get<RPCArg::Optional>(m_fallback); + } +} + +std::string RPCArg::ToDescriptionString() const { std::string ret; ret += "("; @@ -345,6 +403,10 @@ std::string RPCArg::ToDescriptionString(const bool implicitly_required) const ret += "numeric or string"; break; } + case Type::RANGE: { + ret += "numeric or array"; + break; + } case Type::BOOL: { ret += "boolean"; break; @@ -362,19 +424,24 @@ std::string RPCArg::ToDescriptionString(const bool implicitly_required) const // no default case, so the compiler can warn about missing cases } } - if (!implicitly_required) { - ret += ", "; - if (m_optional) { - ret += "optional"; - if (!m_default_value.empty()) { - ret += ", default=" + m_default_value; - } else { - // TODO enable this assert, when all optional parameters have their default value documented - //assert(false); - } - } else { - ret += "required"; - assert(m_default_value.empty()); // Default value is ignored, and must not be present + if (m_fallback.which() == 1) { + ret += ", optional, default=" + boost::get<std::string>(m_fallback); + } else { + switch (boost::get<RPCArg::Optional>(m_fallback)) { + case RPCArg::Optional::OMITTED: { + // nothing to do. Element is treated as if not present and has no default value + break; + } + case RPCArg::Optional::OMITTED_NAMED_ARG: { + ret += ", optional"; // Default value is "null" + break; + } + case RPCArg::Optional::NO: { + ret += ", required"; + break; + } + + // no default case, so the compiler can warn about missing cases } } ret += ")"; @@ -399,6 +466,8 @@ std::string RPCArg::ToStringObj(const bool oneline) const return res + "\"hex\""; case Type::NUM: return res + "n"; + case Type::RANGE: + return res + "n or [n,n]"; case Type::AMOUNT: return res + "amount"; case Type::BOOL: @@ -429,6 +498,7 @@ std::string RPCArg::ToString(const bool oneline) const return "\"" + m_name + "\""; } case Type::NUM: + case Type::RANGE: case Type::AMOUNT: case Type::BOOL: { return m_name; @@ -458,3 +528,17 @@ std::string RPCArg::ToString(const bool oneline) const } assert(false); } + +std::pair<int64_t, int64_t> ParseRange(const UniValue& value) +{ + if (value.isNum()) { + return {0, value.get_int64()}; + } + if (value.isArray() && value.size() == 2 && value[0].isNum() && value[1].isNum()) { + int64_t low = value[0].get_int64(); + int64_t high = value[1].get_int64(); + if (low > high) throw JSONRPCError(RPC_INVALID_PARAMETER, "Range specified as [begin,end] must not have begin after end"); + return {low, high}; + } + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range must be specified as end or as [begin,end]"); +} diff --git a/src/rpc/util.h b/src/rpc/util.h index 4a9d4be787..e4cc1fde44 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -1,17 +1,21 @@ -// Copyright (c) 2017-2018 The Bitcoin Core developers +// Copyright (c) 2017-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_RPC_UTIL_H #define BITCOIN_RPC_UTIL_H +#include <node/transaction.h> #include <pubkey.h> +#include <rpc/protocol.h> #include <script/standard.h> #include <univalue.h> #include <string> #include <vector> +#include <boost/variant.hpp> + class CKeyStore; class CPubKey; class CScript; @@ -28,6 +32,15 @@ CScript CreateMultisigRedeemscript(const int required, const std::vector<CPubKey UniValue DescribeAddress(const CTxDestination& dest); +//! Parse a confirm target option and raise an RPC error if it is invalid. +unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target); + +RPCErrorCode RPCErrorFromTransactionError(TransactionError terr); +UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_string = ""); + +//! Parse a JSON range specified as int64, or [int64, int64] +std::pair<int64_t, int64_t> ParseRange(const UniValue& value); + struct RPCArg { enum class Type { OBJ, @@ -38,12 +51,30 @@ struct RPCArg { OBJ_USER_KEYS, //!< Special type where the user must set the keys e.g. to define multiple addresses; as opposed to e.g. an options object where the keys are predefined AMOUNT, //!< Special type representing a floating point amount (can be either NUM or STR) STR_HEX, //!< Special type that is a STR with only hex chars + RANGE, //!< Special type that is a NUM or [NUM,NUM] }; + + enum class Optional { + /** Required arg */ + NO, + /** + * Optional arg that is a named argument and has a default value of + * `null`. When possible, the default value should be specified. + */ + OMITTED_NAMED_ARG, + /** + * Optional argument with default value omitted because they are + * implicitly clear. That is, elements in an array or object may not + * exist by default. + * When possible, the default value should be specified. + */ + OMITTED, + }; + using Fallback = boost::variant<Optional, /* default value for optional args */ std::string>; const std::string m_name; //!< The name of the arg (can be empty for inner args) const Type m_type; const std::vector<RPCArg> m_inner; //!< Only used for arrays or dicts - const bool m_optional; - const std::string m_default_value; //!< Only used for optional args + const Fallback m_fallback; const std::string m_description; const std::string m_oneline_description; //!< Should be empty unless it is supposed to override the auto-generated summary line const std::vector<std::string> m_type_str; //!< Should be empty unless it is supposed to override the auto-generated type strings. Vector length is either 0 or 2, m_type_str.at(0) will override the type of the value in a key-value pair, m_type_str.at(1) will override the type in the argument description. @@ -51,15 +82,13 @@ struct RPCArg { RPCArg( const std::string& name, const Type& type, - const bool opt, - const std::string& default_val, + const Fallback& fallback, const std::string& description, const std::string& oneline_description = "", const std::vector<std::string>& type_str = {}) : m_name{name}, m_type{type}, - m_optional{opt}, - m_default_value{default_val}, + m_fallback{fallback}, m_description{description}, m_oneline_description{oneline_description}, m_type_str{type_str} @@ -70,8 +99,7 @@ struct RPCArg { RPCArg( const std::string& name, const Type& type, - const bool opt, - const std::string& default_val, + const Fallback& fallback, const std::string& description, const std::vector<RPCArg>& inner, const std::string& oneline_description = "", @@ -79,8 +107,7 @@ struct RPCArg { : m_name{name}, m_type{type}, m_inner{inner}, - m_optional{opt}, - m_default_value{default_val}, + m_fallback{fallback}, m_description{description}, m_oneline_description{oneline_description}, m_type_str{type_str} @@ -88,6 +115,8 @@ struct RPCArg { assert(type == Type::ARR || type == Type::OBJ); } + bool IsOptional() const; + /** * Return the type string of the argument. * Set oneline to allow it to be overridden by a custom oneline type string (m_oneline_description). @@ -101,9 +130,8 @@ struct RPCArg { /** * Return the description string, including the argument type and whether * the argument is required. - * implicitly_required is set for arguments in an array, which are neither optional nor required. */ - std::string ToDescriptionString(bool implicitly_required = false) const; + std::string ToDescriptionString() const; }; struct RPCResult { @@ -164,6 +192,8 @@ public: RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples); std::string ToString() const; + /** If the supplied number of args is neither too small nor too high */ + bool IsValidNumArgs(size_t num_args) const; private: const std::string m_name; diff --git a/src/scheduler.h b/src/scheduler.h index 6d7f42cf9f..436f661c59 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -45,13 +45,13 @@ public: // Call func at/after time t void schedule(Function f, boost::chrono::system_clock::time_point t=boost::chrono::system_clock::now()); - // Convenience method: call f once deltaSeconds from now + // Convenience method: call f once deltaMilliSeconds from now void scheduleFromNow(Function f, int64_t deltaMilliSeconds); // Another convenience method: call f approximately - // every deltaSeconds forever, starting deltaSeconds from now. + // every deltaMilliSeconds forever, starting deltaMilliSeconds from now. // To be more precise: every time f is finished, it - // is rescheduled to run deltaSeconds later. If you + // is rescheduled to run deltaMilliSeconds later. If you // need more accurate scheduling, don't use this method. void scheduleEvery(Function f, int64_t deltaMilliSeconds); diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 41e0f2e117..43448d7222 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -10,6 +10,7 @@ #include <script/standard.h> #include <span.h> +#include <util/bip32.h> #include <util/system.h> #include <util/strencodings.h> @@ -20,21 +21,130 @@ namespace { //////////////////////////////////////////////////////////////////////////// -// Internal representation // +// Checksum // //////////////////////////////////////////////////////////////////////////// -typedef std::vector<uint32_t> KeyPath; +// This section implements a checksum algorithm for descriptors with the +// following properties: +// * Mistakes in a descriptor string are measured in "symbol errors". The higher +// the number of symbol errors, the harder it is to detect: +// * An error substituting a character from 0123456789()[],'/*abcdefgh@:$%{} for +// another in that set always counts as 1 symbol error. +// * Note that hex encoded keys are covered by these characters. Xprvs and +// xpubs use other characters too, but already have their own checksum +// mechanism. +// * Function names like "multi()" use other characters, but mistakes in +// these would generally result in an unparseable descriptor. +// * A case error always counts as 1 symbol error. +// * Any other 1 character substitution error counts as 1 or 2 symbol errors. +// * Any 1 symbol error is always detected. +// * Any 2 or 3 symbol error in a descriptor of up to 49154 characters is always detected. +// * Any 4 symbol error in a descriptor of up to 507 characters is always detected. +// * Any 5 symbol error in a descriptor of up to 77 characters is always detected. +// * Is optimized to minimize the chance a 5 symbol error in a descriptor up to 387 characters is undetected +// * Random errors have a chance of 1 in 2**40 of being undetected. +// +// These properties are achieved by expanding every group of 3 (non checksum) characters into +// 4 GF(32) symbols, over which a cyclic code is defined. + +/* + * Interprets c as 8 groups of 5 bits which are the coefficients of a degree 8 polynomial over GF(32), + * multiplies that polynomial by x, computes its remainder modulo a generator, and adds the constant term val. + * + * This generator is G(x) = x^8 + {30}x^7 + {23}x^6 + {15}x^5 + {14}x^4 + {10}x^3 + {6}x^2 + {12}x + {9}. + * It is chosen to define an cyclic error detecting code which is selected by: + * - Starting from all BCH codes over GF(32) of degree 8 and below, which by construction guarantee detecting + * 3 errors in windows up to 19000 symbols. + * - Taking all those generators, and for degree 7 ones, extend them to degree 8 by adding all degree-1 factors. + * - Selecting just the set of generators that guarantee detecting 4 errors in a window of length 512. + * - Selecting one of those with best worst-case behavior for 5 errors in windows of length up to 512. + * + * The generator and the constants to implement it can be verified using this Sage code: + * B = GF(2) # Binary field + * BP.<b> = B[] # Polynomials over the binary field + * F_mod = b**5 + b**3 + 1 + * F.<f> = GF(32, modulus=F_mod, repr='int') # GF(32) definition + * FP.<x> = F[] # Polynomials over GF(32) + * E_mod = x**3 + x + F.fetch_int(8) + * E.<e> = F.extension(E_mod) # Extension field definition + * alpha = e**2743 # Choice of an element in extension field + * for p in divisors(E.order() - 1): # Verify alpha has order 32767. + * assert((alpha**p == 1) == (p % 32767 == 0)) + * G = lcm([(alpha**i).minpoly() for i in [1056,1057,1058]] + [x + 1]) + * print(G) # Print out the generator + * for i in [1,2,4,8,16]: # Print out {1,2,4,8,16}*(G mod x^8), packed in hex integers. + * v = 0 + * for coef in reversed((F.fetch_int(i)*(G % x**8)).coefficients(sparse=True)): + * v = v*32 + coef.integer_representation() + * print("0x%x" % v) + */ +uint64_t PolyMod(uint64_t c, int val) +{ + uint8_t c0 = c >> 35; + c = ((c & 0x7ffffffff) << 5) ^ val; + if (c0 & 1) c ^= 0xf5dee51989; + if (c0 & 2) c ^= 0xa9fdca3312; + if (c0 & 4) c ^= 0x1bab10e32d; + if (c0 & 8) c ^= 0x3706b1677a; + if (c0 & 16) c ^= 0x644d626ffd; + return c; +} -std::string FormatKeyPath(const KeyPath& path) +std::string DescriptorChecksum(const Span<const char>& span) { - std::string ret; - for (auto i : path) { - ret += strprintf("/%i", (i << 1) >> 1); - if (i >> 31) ret += '\''; + /** A character set designed such that: + * - The most common 'unprotected' descriptor characters (hex, keypaths) are in the first group of 32. + * - Case errors cause an offset that's a multiple of 32. + * - As many alphabetic characters are in the same group (while following the above restrictions). + * + * If p(x) gives the position of a character c in this character set, every group of 3 characters + * (a,b,c) is encoded as the 4 symbols (p(a) & 31, p(b) & 31, p(c) & 31, (p(a) / 32) + 3 * (p(b) / 32) + 9 * (p(c) / 32). + * This means that changes that only affect the lower 5 bits of the position, or only the higher 2 bits, will just + * affect a single symbol. + * + * As a result, within-group-of-32 errors count as 1 symbol, as do cross-group errors that don't affect + * the position within the groups. + */ + static std::string INPUT_CHARSET = + "0123456789()[],'/*abcdefgh@:$%{}" + "IJKLMNOPQRSTUVWXYZ&+-.;<=>?!^_|~" + "ijklmnopqrstuvwxyzABCDEFGH`#\"\\ "; + + /** The character set for the checksum itself (same as bech32). */ + static std::string CHECKSUM_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"; + + uint64_t c = 1; + int cls = 0; + int clscount = 0; + for (auto ch : span) { + auto pos = INPUT_CHARSET.find(ch); + if (pos == std::string::npos) return ""; + c = PolyMod(c, pos & 31); // Emit a symbol for the position inside the group, for every character. + cls = cls * 3 + (pos >> 5); // Accumulate the group numbers + if (++clscount == 3) { + // Emit an extra symbol representing the group numbers, for every 3 characters. + c = PolyMod(c, cls); + cls = 0; + clscount = 0; + } } + if (clscount > 0) c = PolyMod(c, cls); + for (int j = 0; j < 8; ++j) c = PolyMod(c, 0); // Shift further to determine the checksum. + c ^= 1; // Prevent appending zeroes from not affecting the checksum. + + std::string ret(8, ' '); + for (int j = 0; j < 8; ++j) ret[j] = CHECKSUM_CHARSET[(c >> (5 * (7 - j))) & 31]; return ret; } +std::string AddChecksum(const std::string& str) { return str + "#" + DescriptorChecksum(MakeSpan(str)); } + +//////////////////////////////////////////////////////////////////////////// +// Internal representation // +//////////////////////////////////////////////////////////////////////////// + +typedef std::vector<uint32_t> KeyPath; + /** Interface for public key objects in descriptors. */ struct PubkeyProvider { @@ -63,7 +173,7 @@ class OriginPubkeyProvider final : public PubkeyProvider std::string OriginString() const { - return HexStr(std::begin(m_origin.fingerprint), std::end(m_origin.fingerprint)) + FormatKeyPath(m_origin.path); + return HexStr(std::begin(m_origin.fingerprint), std::end(m_origin.fingerprint)) + FormatHDKeypath(m_origin.path); } public: @@ -184,7 +294,7 @@ public: } std::string ToString() const override { - std::string ret = EncodeExtPubKey(m_extkey) + FormatKeyPath(m_path); + std::string ret = EncodeExtPubKey(m_extkey) + FormatHDKeypath(m_path); if (IsRange()) { ret += "/*"; if (m_derive == DeriveType::HARDENED) ret += '\''; @@ -195,7 +305,7 @@ public: { CExtKey key; if (!GetExtKey(arg, key)) return false; - out = EncodeExtKey(key) + FormatKeyPath(m_path); + out = EncodeExtKey(key) + FormatHDKeypath(m_path); if (IsRange()) { out += "/*"; if (m_derive == DeriveType::HARDENED) out += '\''; @@ -282,10 +392,15 @@ public: { std::string ret; ToStringHelper(nullptr, ret, false); - return ret; + return AddChecksum(ret); } - bool ToPrivateString(const SigningProvider& arg, std::string& out) const override final { return ToStringHelper(&arg, out, true); } + bool ToPrivateString(const SigningProvider& arg, std::string& out) const override final + { + bool ret = ToStringHelper(&arg, out, true); + out = AddChecksum(out); + return ret; + } bool ExpandHelper(int pos, const SigningProvider& arg, Span<const unsigned char>* cache_read, std::vector<CScript>& output_scripts, FlatSigningProvider& out, std::vector<unsigned char>* cache_write) const { @@ -760,11 +875,25 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo return MakeUnique<RawDescriptor>(script); } + } // namespace -std::unique_ptr<Descriptor> Parse(const std::string& descriptor, FlatSigningProvider& out) +std::unique_ptr<Descriptor> Parse(const std::string& descriptor, FlatSigningProvider& out, bool require_checksum) { Span<const char> sp(descriptor.data(), descriptor.size()); + + // Checksum checks + auto check_split = Split(sp, '#'); + if (check_split.size() > 2) return nullptr; // Multiple '#' symbols + if (check_split.size() == 1 && require_checksum) return nullptr; // Missing checksum + if (check_split.size() == 2) { + if (check_split[1].size() != 8) return nullptr; // Unexpected length for checksum + auto checksum = DescriptorChecksum(check_split[0]); + if (checksum.empty()) return nullptr; // Invalid characters in payload + if (!std::equal(checksum.begin(), checksum.end(), check_split[1].begin())) return nullptr; // Checksum mismatch + } + sp = check_split[0]; + auto ret = ParseScript(sp, ParseScriptContext::TOP, out); if (sp.size() == 0 && ret) return std::unique_ptr<Descriptor>(std::move(ret)); return nullptr; diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 44f0efca03..907a102284 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -62,8 +62,15 @@ struct Descriptor { virtual bool ExpandFromCache(int pos, const std::vector<unsigned char>& cache, std::vector<CScript>& output_scripts, FlatSigningProvider& out) const = 0; }; -/** Parse a descriptor string. Included private keys are put in out. Returns nullptr if parsing fails. */ -std::unique_ptr<Descriptor> Parse(const std::string& descriptor, FlatSigningProvider& out); +/** Parse a descriptor string. Included private keys are put in out. + * + * If the descriptor has a checksum, it must be valid. If require_checksum + * is set, the checksum is mandatory - otherwise it is optional. + * + * If a parse error occurs, or the checksum is missing/invalid, or anything + * else is wrong, nullptr is returned. + */ +std::unique_ptr<Descriptor> Parse(const std::string& descriptor, FlatSigningProvider& out, bool require_checksum = false); /** Find a descriptor for the specified script, using information from provider where possible. * diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 792fb2997f..320956d0c4 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -83,6 +83,8 @@ static bool CreateSig(const BaseSignatureCreator& creator, SignatureData& sigdat assert(i.second); return true; } + // Could not make signature or signature not found, add keyid to missing + sigdata.missing_sigs.push_back(keyid); return false; } @@ -116,17 +118,24 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator case TX_PUBKEYHASH: { CKeyID keyID = CKeyID(uint160(vSolutions[0])); CPubKey pubkey; - if (!GetPubKey(provider, sigdata, keyID, pubkey)) return false; + if (!GetPubKey(provider, sigdata, keyID, pubkey)) { + // Pubkey could not be found, add to missing + sigdata.missing_pubkeys.push_back(keyID); + return false; + } if (!CreateSig(creator, sigdata, provider, sig, pubkey, scriptPubKey, sigversion)) return false; ret.push_back(std::move(sig)); ret.push_back(ToByteVector(pubkey)); return true; } case TX_SCRIPTHASH: - if (GetCScript(provider, sigdata, uint160(vSolutions[0]), scriptRet)) { + h160 = uint160(vSolutions[0]); + if (GetCScript(provider, sigdata, h160, scriptRet)) { ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); return true; } + // Could not find redeemScript, add to missing + sigdata.missing_redeem_script = h160; return false; case TX_MULTISIG: { @@ -154,6 +163,8 @@ static bool SignStep(const SigningProvider& provider, const BaseSignatureCreator ret.push_back(std::vector<unsigned char>(scriptRet.begin(), scriptRet.end())); return true; } + // Could not find witnessScript, add to missing + sigdata.missing_witness_script = uint256(vSolutions[0]); return false; default: @@ -232,67 +243,6 @@ bool ProduceSignature(const SigningProvider& provider, const BaseSignatureCreato return sigdata.complete; } -bool PSBTInputSigned(PSBTInput& input) -{ - return !input.final_script_sig.empty() || !input.final_script_witness.IsNull(); -} - -bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash) -{ - PSBTInput& input = psbt.inputs.at(index); - const CMutableTransaction& tx = *psbt.tx; - - if (PSBTInputSigned(input)) { - return true; - } - - // Fill SignatureData with input info - SignatureData sigdata; - input.FillSignatureData(sigdata); - - // Get UTXO - bool require_witness_sig = false; - CTxOut utxo; - - // Verify input sanity, which checks that at most one of witness or non-witness utxos is provided. - if (!input.IsSane()) { - return false; - } - - if (input.non_witness_utxo) { - // If we're taking our information from a non-witness UTXO, verify that it matches the prevout. - COutPoint prevout = tx.vin[index].prevout; - if (input.non_witness_utxo->GetHash() != prevout.hash) { - return false; - } - utxo = input.non_witness_utxo->vout[prevout.n]; - } else if (!input.witness_utxo.IsNull()) { - utxo = input.witness_utxo; - // When we're taking our information from a witness UTXO, we can't verify it is actually data from - // the output being spent. This is safe in case a witness signature is produced (which includes this - // information directly in the hash), but not for non-witness signatures. Remember that we require - // a witness signature in this situation. - require_witness_sig = true; - } else { - return false; - } - - MutableTransactionSignatureCreator creator(&tx, index, utxo.nValue, sighash); - sigdata.witness = false; - bool sig_complete = ProduceSignature(provider, creator, utxo.scriptPubKey, sigdata); - // Verify that a witness signature was produced in case one was required. - if (require_witness_sig && !sigdata.witness) return false; - input.FromSignatureData(sigdata); - - // If we have a witness signature, use the smaller witness UTXO. - if (sigdata.witness) { - input.witness_utxo = utxo; - input.non_witness_utxo = nullptr; - } - - return sig_complete; -} - class SignatureExtractorChecker final : public BaseSignatureChecker { private: @@ -509,166 +459,6 @@ bool IsSolvable(const SigningProvider& provider, const CScript& script) return false; } -PartiallySignedTransaction::PartiallySignedTransaction(const CMutableTransaction& tx) : tx(tx) -{ - inputs.resize(tx.vin.size()); - outputs.resize(tx.vout.size()); -} - -bool PartiallySignedTransaction::IsNull() const -{ - return !tx && inputs.empty() && outputs.empty() && unknown.empty(); -} - -void PartiallySignedTransaction::Merge(const PartiallySignedTransaction& psbt) -{ - for (unsigned int i = 0; i < inputs.size(); ++i) { - inputs[i].Merge(psbt.inputs[i]); - } - for (unsigned int i = 0; i < outputs.size(); ++i) { - outputs[i].Merge(psbt.outputs[i]); - } - unknown.insert(psbt.unknown.begin(), psbt.unknown.end()); -} - -bool PartiallySignedTransaction::IsSane() const -{ - for (PSBTInput input : inputs) { - if (!input.IsSane()) return false; - } - return true; -} - -bool PSBTInput::IsNull() const -{ - return !non_witness_utxo && witness_utxo.IsNull() && partial_sigs.empty() && unknown.empty() && hd_keypaths.empty() && redeem_script.empty() && witness_script.empty(); -} - -void PSBTInput::FillSignatureData(SignatureData& sigdata) const -{ - if (!final_script_sig.empty()) { - sigdata.scriptSig = final_script_sig; - sigdata.complete = true; - } - if (!final_script_witness.IsNull()) { - sigdata.scriptWitness = final_script_witness; - sigdata.complete = true; - } - if (sigdata.complete) { - return; - } - - sigdata.signatures.insert(partial_sigs.begin(), partial_sigs.end()); - if (!redeem_script.empty()) { - sigdata.redeem_script = redeem_script; - } - if (!witness_script.empty()) { - sigdata.witness_script = witness_script; - } - for (const auto& key_pair : hd_keypaths) { - sigdata.misc_pubkeys.emplace(key_pair.first.GetID(), key_pair); - } -} - -void PSBTInput::FromSignatureData(const SignatureData& sigdata) -{ - if (sigdata.complete) { - partial_sigs.clear(); - hd_keypaths.clear(); - redeem_script.clear(); - witness_script.clear(); - - if (!sigdata.scriptSig.empty()) { - final_script_sig = sigdata.scriptSig; - } - if (!sigdata.scriptWitness.IsNull()) { - final_script_witness = sigdata.scriptWitness; - } - return; - } - - partial_sigs.insert(sigdata.signatures.begin(), sigdata.signatures.end()); - if (redeem_script.empty() && !sigdata.redeem_script.empty()) { - redeem_script = sigdata.redeem_script; - } - if (witness_script.empty() && !sigdata.witness_script.empty()) { - witness_script = sigdata.witness_script; - } - for (const auto& entry : sigdata.misc_pubkeys) { - hd_keypaths.emplace(entry.second); - } -} - -void PSBTInput::Merge(const PSBTInput& input) -{ - if (!non_witness_utxo && input.non_witness_utxo) non_witness_utxo = input.non_witness_utxo; - if (witness_utxo.IsNull() && !input.witness_utxo.IsNull()) { - witness_utxo = input.witness_utxo; - non_witness_utxo = nullptr; // Clear out any non-witness utxo when we set a witness one. - } - - partial_sigs.insert(input.partial_sigs.begin(), input.partial_sigs.end()); - hd_keypaths.insert(input.hd_keypaths.begin(), input.hd_keypaths.end()); - unknown.insert(input.unknown.begin(), input.unknown.end()); - - if (redeem_script.empty() && !input.redeem_script.empty()) redeem_script = input.redeem_script; - if (witness_script.empty() && !input.witness_script.empty()) witness_script = input.witness_script; - if (final_script_sig.empty() && !input.final_script_sig.empty()) final_script_sig = input.final_script_sig; - if (final_script_witness.IsNull() && !input.final_script_witness.IsNull()) final_script_witness = input.final_script_witness; -} - -bool PSBTInput::IsSane() const -{ - // Cannot have both witness and non-witness utxos - if (!witness_utxo.IsNull() && non_witness_utxo) return false; - - // If we have a witness_script or a scriptWitness, we must also have a witness utxo - if (!witness_script.empty() && witness_utxo.IsNull()) return false; - if (!final_script_witness.IsNull() && witness_utxo.IsNull()) return false; - - return true; -} - -void PSBTOutput::FillSignatureData(SignatureData& sigdata) const -{ - if (!redeem_script.empty()) { - sigdata.redeem_script = redeem_script; - } - if (!witness_script.empty()) { - sigdata.witness_script = witness_script; - } - for (const auto& key_pair : hd_keypaths) { - sigdata.misc_pubkeys.emplace(key_pair.first.GetID(), key_pair); - } -} - -void PSBTOutput::FromSignatureData(const SignatureData& sigdata) -{ - if (redeem_script.empty() && !sigdata.redeem_script.empty()) { - redeem_script = sigdata.redeem_script; - } - if (witness_script.empty() && !sigdata.witness_script.empty()) { - witness_script = sigdata.witness_script; - } - for (const auto& entry : sigdata.misc_pubkeys) { - hd_keypaths.emplace(entry.second); - } -} - -bool PSBTOutput::IsNull() const -{ - return redeem_script.empty() && witness_script.empty() && hd_keypaths.empty() && unknown.empty(); -} - -void PSBTOutput::Merge(const PSBTOutput& output) -{ - hd_keypaths.insert(output.hd_keypaths.begin(), output.hd_keypaths.end()); - unknown.insert(output.unknown.begin(), output.unknown.end()); - - if (redeem_script.empty() && !output.redeem_script.empty()) redeem_script = output.redeem_script; - if (witness_script.empty() && !output.witness_script.empty()) witness_script = output.witness_script; -} - bool HidingSigningProvider::GetCScript(const CScriptID& scriptid, CScript& script) const { return m_provider->GetCScript(scriptid, script); diff --git a/src/script/sign.h b/src/script/sign.h index e884f4c480..491fb54c45 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -22,13 +22,27 @@ struct CMutableTransaction; struct KeyOriginInfo { - unsigned char fingerprint[4]; + unsigned char fingerprint[4]; //!< First 32 bits of the Hash160 of the public key at the root of the path std::vector<uint32_t> path; friend bool operator==(const KeyOriginInfo& a, const KeyOriginInfo& b) { return std::equal(std::begin(a.fingerprint), std::end(a.fingerprint), std::begin(b.fingerprint)) && a.path == b.path; } + + ADD_SERIALIZE_METHODS; + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(fingerprint); + READWRITE(path); + } + + void clear() + { + memset(fingerprint, 0, 4); + path.clear(); + } }; /** An interface to be implemented by keystores that support signing. */ @@ -117,38 +131,16 @@ struct SignatureData { CScriptWitness scriptWitness; ///< The scriptWitness of an input. Contains complete signatures or the traditional partial signatures format. scriptWitness is part of a transaction input per BIP 144. std::map<CKeyID, SigPair> signatures; ///< BIP 174 style partial signatures for the input. May contain all signatures necessary for producing a final scriptSig or scriptWitness. std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>> misc_pubkeys; + std::vector<CKeyID> missing_pubkeys; ///< KeyIDs of pubkeys which could not be found + std::vector<CKeyID> missing_sigs; ///< KeyIDs of pubkeys for signatures which could not be found + uint160 missing_redeem_script; ///< ScriptID of the missing redeemScript (if any) + uint256 missing_witness_script; ///< SHA256 of the missing witnessScript (if any) SignatureData() {} explicit SignatureData(const CScript& script) : scriptSig(script) {} void MergeSignatureData(SignatureData sigdata); }; -// Magic bytes -static constexpr uint8_t PSBT_MAGIC_BYTES[5] = {'p', 's', 'b', 't', 0xff}; - -// Global types -static constexpr uint8_t PSBT_GLOBAL_UNSIGNED_TX = 0x00; - -// Input types -static constexpr uint8_t PSBT_IN_NON_WITNESS_UTXO = 0x00; -static constexpr uint8_t PSBT_IN_WITNESS_UTXO = 0x01; -static constexpr uint8_t PSBT_IN_PARTIAL_SIG = 0x02; -static constexpr uint8_t PSBT_IN_SIGHASH = 0x03; -static constexpr uint8_t PSBT_IN_REDEEMSCRIPT = 0x04; -static constexpr uint8_t PSBT_IN_WITNESSSCRIPT = 0x05; -static constexpr uint8_t PSBT_IN_BIP32_DERIVATION = 0x06; -static constexpr uint8_t PSBT_IN_SCRIPTSIG = 0x07; -static constexpr uint8_t PSBT_IN_SCRIPTWITNESS = 0x08; - -// Output types -static constexpr uint8_t PSBT_OUT_REDEEMSCRIPT = 0x00; -static constexpr uint8_t PSBT_OUT_WITNESSSCRIPT = 0x01; -static constexpr uint8_t PSBT_OUT_BIP32_DERIVATION = 0x02; - -// The separator is 0x00. Reading this in means that the unserializer can interpret it -// as a 0 length key which indicates that this is the separator. The separator has no value. -static constexpr uint8_t PSBT_SEPARATOR = 0x00; - // Takes a stream and multiple arguments and serializes them as if first serialized into a vector and then into the stream // The resulting output into the stream has the total serialized length of all of the objects followed by all objects concatenated with each other. template<typename Stream, typename... X> @@ -223,514 +215,6 @@ void SerializeHDKeypaths(Stream& s, const std::map<CPubKey, KeyOriginInfo>& hd_k } } -/** A structure for PSBTs which contain per-input information */ -struct PSBTInput -{ - CTransactionRef non_witness_utxo; - CTxOut witness_utxo; - CScript redeem_script; - CScript witness_script; - CScript final_script_sig; - CScriptWitness final_script_witness; - std::map<CPubKey, KeyOriginInfo> hd_keypaths; - std::map<CKeyID, SigPair> partial_sigs; - std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; - int sighash_type = 0; - - bool IsNull() const; - void FillSignatureData(SignatureData& sigdata) const; - void FromSignatureData(const SignatureData& sigdata); - void Merge(const PSBTInput& input); - bool IsSane() const; - PSBTInput() {} - - template <typename Stream> - inline void Serialize(Stream& s) const { - // Write the utxo - // If there is a non-witness utxo, then don't add the witness one. - if (non_witness_utxo) { - SerializeToVector(s, PSBT_IN_NON_WITNESS_UTXO); - OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); - SerializeToVector(os, non_witness_utxo); - } else if (!witness_utxo.IsNull()) { - SerializeToVector(s, PSBT_IN_WITNESS_UTXO); - SerializeToVector(s, witness_utxo); - } - - if (final_script_sig.empty() && final_script_witness.IsNull()) { - // Write any partial signatures - for (auto sig_pair : partial_sigs) { - SerializeToVector(s, PSBT_IN_PARTIAL_SIG, MakeSpan(sig_pair.second.first)); - s << sig_pair.second.second; - } - - // Write the sighash type - if (sighash_type > 0) { - SerializeToVector(s, PSBT_IN_SIGHASH); - SerializeToVector(s, sighash_type); - } - - // Write the redeem script - if (!redeem_script.empty()) { - SerializeToVector(s, PSBT_IN_REDEEMSCRIPT); - s << redeem_script; - } - - // Write the witness script - if (!witness_script.empty()) { - SerializeToVector(s, PSBT_IN_WITNESSSCRIPT); - s << witness_script; - } - - // Write any hd keypaths - SerializeHDKeypaths(s, hd_keypaths, PSBT_IN_BIP32_DERIVATION); - } - - // Write script sig - if (!final_script_sig.empty()) { - SerializeToVector(s, PSBT_IN_SCRIPTSIG); - s << final_script_sig; - } - // write script witness - if (!final_script_witness.IsNull()) { - SerializeToVector(s, PSBT_IN_SCRIPTWITNESS); - SerializeToVector(s, final_script_witness.stack); - } - - // Write unknown things - for (auto& entry : unknown) { - s << entry.first; - s << entry.second; - } - - s << PSBT_SEPARATOR; - } - - - template <typename Stream> - inline void Unserialize(Stream& s) { - // Read loop - bool found_sep = false; - while(!s.empty()) { - // Read - std::vector<unsigned char> key; - s >> key; - - // the key is empty if that was actually a separator byte - // This is a special case for key lengths 0 as those are not allowed (except for separator) - if (key.empty()) { - found_sep = true; - break; - } - - // First byte of key is the type - unsigned char type = key[0]; - - // Do stuff based on type - switch(type) { - case PSBT_IN_NON_WITNESS_UTXO: - { - if (non_witness_utxo) { - throw std::ios_base::failure("Duplicate Key, input non-witness utxo already provided"); - } else if (key.size() != 1) { - throw std::ios_base::failure("Non-witness utxo key is more than one byte type"); - } - // Set the stream to unserialize with witness since this is always a valid network transaction - OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() & ~SERIALIZE_TRANSACTION_NO_WITNESS); - UnserializeFromVector(os, non_witness_utxo); - break; - } - case PSBT_IN_WITNESS_UTXO: - if (!witness_utxo.IsNull()) { - throw std::ios_base::failure("Duplicate Key, input witness utxo already provided"); - } else if (key.size() != 1) { - throw std::ios_base::failure("Witness utxo key is more than one byte type"); - } - UnserializeFromVector(s, witness_utxo); - break; - case PSBT_IN_PARTIAL_SIG: - { - // Make sure that the key is the size of pubkey + 1 - if (key.size() != CPubKey::PUBLIC_KEY_SIZE + 1 && key.size() != CPubKey::COMPRESSED_PUBLIC_KEY_SIZE + 1) { - throw std::ios_base::failure("Size of key was not the expected size for the type partial signature pubkey"); - } - // Read in the pubkey from key - CPubKey pubkey(key.begin() + 1, key.end()); - if (!pubkey.IsFullyValid()) { - throw std::ios_base::failure("Invalid pubkey"); - } - if (partial_sigs.count(pubkey.GetID()) > 0) { - throw std::ios_base::failure("Duplicate Key, input partial signature for pubkey already provided"); - } - - // Read in the signature from value - std::vector<unsigned char> sig; - s >> sig; - - // Add to list - partial_sigs.emplace(pubkey.GetID(), SigPair(pubkey, std::move(sig))); - break; - } - case PSBT_IN_SIGHASH: - if (sighash_type > 0) { - throw std::ios_base::failure("Duplicate Key, input sighash type already provided"); - } else if (key.size() != 1) { - throw std::ios_base::failure("Sighash type key is more than one byte type"); - } - UnserializeFromVector(s, sighash_type); - break; - case PSBT_IN_REDEEMSCRIPT: - { - if (!redeem_script.empty()) { - throw std::ios_base::failure("Duplicate Key, input redeemScript already provided"); - } else if (key.size() != 1) { - throw std::ios_base::failure("Input redeemScript key is more than one byte type"); - } - s >> redeem_script; - break; - } - case PSBT_IN_WITNESSSCRIPT: - { - if (!witness_script.empty()) { - throw std::ios_base::failure("Duplicate Key, input witnessScript already provided"); - } else if (key.size() != 1) { - throw std::ios_base::failure("Input witnessScript key is more than one byte type"); - } - s >> witness_script; - break; - } - case PSBT_IN_BIP32_DERIVATION: - { - DeserializeHDKeypaths(s, key, hd_keypaths); - break; - } - case PSBT_IN_SCRIPTSIG: - { - if (!final_script_sig.empty()) { - throw std::ios_base::failure("Duplicate Key, input final scriptSig already provided"); - } else if (key.size() != 1) { - throw std::ios_base::failure("Final scriptSig key is more than one byte type"); - } - s >> final_script_sig; - break; - } - case PSBT_IN_SCRIPTWITNESS: - { - if (!final_script_witness.IsNull()) { - throw std::ios_base::failure("Duplicate Key, input final scriptWitness already provided"); - } else if (key.size() != 1) { - throw std::ios_base::failure("Final scriptWitness key is more than one byte type"); - } - UnserializeFromVector(s, final_script_witness.stack); - break; - } - // Unknown stuff - default: - if (unknown.count(key) > 0) { - throw std::ios_base::failure("Duplicate Key, key for unknown value already provided"); - } - // Read in the value - std::vector<unsigned char> val_bytes; - s >> val_bytes; - unknown.emplace(std::move(key), std::move(val_bytes)); - break; - } - } - - if (!found_sep) { - throw std::ios_base::failure("Separator is missing at the end of an input map"); - } - } - - template <typename Stream> - PSBTInput(deserialize_type, Stream& s) { - Unserialize(s); - } -}; - -/** A structure for PSBTs which contains per output information */ -struct PSBTOutput -{ - CScript redeem_script; - CScript witness_script; - std::map<CPubKey, KeyOriginInfo> hd_keypaths; - std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; - - bool IsNull() const; - void FillSignatureData(SignatureData& sigdata) const; - void FromSignatureData(const SignatureData& sigdata); - void Merge(const PSBTOutput& output); - bool IsSane() const; - PSBTOutput() {} - - template <typename Stream> - inline void Serialize(Stream& s) const { - // Write the redeem script - if (!redeem_script.empty()) { - SerializeToVector(s, PSBT_OUT_REDEEMSCRIPT); - s << redeem_script; - } - - // Write the witness script - if (!witness_script.empty()) { - SerializeToVector(s, PSBT_OUT_WITNESSSCRIPT); - s << witness_script; - } - - // Write any hd keypaths - SerializeHDKeypaths(s, hd_keypaths, PSBT_OUT_BIP32_DERIVATION); - - // Write unknown things - for (auto& entry : unknown) { - s << entry.first; - s << entry.second; - } - - s << PSBT_SEPARATOR; - } - - - template <typename Stream> - inline void Unserialize(Stream& s) { - // Read loop - bool found_sep = false; - while(!s.empty()) { - // Read - std::vector<unsigned char> key; - s >> key; - - // the key is empty if that was actually a separator byte - // This is a special case for key lengths 0 as those are not allowed (except for separator) - if (key.empty()) { - found_sep = true; - break; - } - - // First byte of key is the type - unsigned char type = key[0]; - - // Do stuff based on type - switch(type) { - case PSBT_OUT_REDEEMSCRIPT: - { - if (!redeem_script.empty()) { - throw std::ios_base::failure("Duplicate Key, output redeemScript already provided"); - } else if (key.size() != 1) { - throw std::ios_base::failure("Output redeemScript key is more than one byte type"); - } - s >> redeem_script; - break; - } - case PSBT_OUT_WITNESSSCRIPT: - { - if (!witness_script.empty()) { - throw std::ios_base::failure("Duplicate Key, output witnessScript already provided"); - } else if (key.size() != 1) { - throw std::ios_base::failure("Output witnessScript key is more than one byte type"); - } - s >> witness_script; - break; - } - case PSBT_OUT_BIP32_DERIVATION: - { - DeserializeHDKeypaths(s, key, hd_keypaths); - break; - } - // Unknown stuff - default: { - if (unknown.count(key) > 0) { - throw std::ios_base::failure("Duplicate Key, key for unknown value already provided"); - } - // Read in the value - std::vector<unsigned char> val_bytes; - s >> val_bytes; - unknown.emplace(std::move(key), std::move(val_bytes)); - break; - } - } - } - - if (!found_sep) { - throw std::ios_base::failure("Separator is missing at the end of an output map"); - } - } - - template <typename Stream> - PSBTOutput(deserialize_type, Stream& s) { - Unserialize(s); - } -}; - -/** A version of CTransaction with the PSBT format*/ -struct PartiallySignedTransaction -{ - boost::optional<CMutableTransaction> tx; - std::vector<PSBTInput> inputs; - std::vector<PSBTOutput> outputs; - std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; - - bool IsNull() const; - void Merge(const PartiallySignedTransaction& psbt); - bool IsSane() const; - PartiallySignedTransaction() {} - PartiallySignedTransaction(const PartiallySignedTransaction& psbt_in) : tx(psbt_in.tx), inputs(psbt_in.inputs), outputs(psbt_in.outputs), unknown(psbt_in.unknown) {} - explicit PartiallySignedTransaction(const CMutableTransaction& tx); - - // Only checks if they refer to the same transaction - friend bool operator==(const PartiallySignedTransaction& a, const PartiallySignedTransaction &b) - { - return a.tx->GetHash() == b.tx->GetHash(); - } - friend bool operator!=(const PartiallySignedTransaction& a, const PartiallySignedTransaction &b) - { - return !(a == b); - } - - template <typename Stream> - inline void Serialize(Stream& s) const { - - // magic bytes - s << PSBT_MAGIC_BYTES; - - // unsigned tx flag - SerializeToVector(s, PSBT_GLOBAL_UNSIGNED_TX); - - // Write serialized tx to a stream - OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); - SerializeToVector(os, *tx); - - // Write the unknown things - for (auto& entry : unknown) { - s << entry.first; - s << entry.second; - } - - // Separator - s << PSBT_SEPARATOR; - - // Write inputs - for (const PSBTInput& input : inputs) { - s << input; - } - // Write outputs - for (const PSBTOutput& output : outputs) { - s << output; - } - } - - - template <typename Stream> - inline void Unserialize(Stream& s) { - // Read the magic bytes - uint8_t magic[5]; - s >> magic; - if (!std::equal(magic, magic + 5, PSBT_MAGIC_BYTES)) { - throw std::ios_base::failure("Invalid PSBT magic bytes"); - } - - // Read global data - bool found_sep = false; - while(!s.empty()) { - // Read - std::vector<unsigned char> key; - s >> key; - - // the key is empty if that was actually a separator byte - // This is a special case for key lengths 0 as those are not allowed (except for separator) - if (key.empty()) { - found_sep = true; - break; - } - - // First byte of key is the type - unsigned char type = key[0]; - - // Do stuff based on type - switch(type) { - case PSBT_GLOBAL_UNSIGNED_TX: - { - if (tx) { - throw std::ios_base::failure("Duplicate Key, unsigned tx already provided"); - } else if (key.size() != 1) { - throw std::ios_base::failure("Global unsigned tx key is more than one byte type"); - } - CMutableTransaction mtx; - // Set the stream to serialize with non-witness since this should always be non-witness - OverrideStream<Stream> os(&s, s.GetType(), s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS); - UnserializeFromVector(os, mtx); - tx = std::move(mtx); - // Make sure that all scriptSigs and scriptWitnesses are empty - for (const CTxIn& txin : tx->vin) { - if (!txin.scriptSig.empty() || !txin.scriptWitness.IsNull()) { - throw std::ios_base::failure("Unsigned tx does not have empty scriptSigs and scriptWitnesses."); - } - } - break; - } - // Unknown stuff - default: { - if (unknown.count(key) > 0) { - throw std::ios_base::failure("Duplicate Key, key for unknown value already provided"); - } - // Read in the value - std::vector<unsigned char> val_bytes; - s >> val_bytes; - unknown.emplace(std::move(key), std::move(val_bytes)); - } - } - } - - if (!found_sep) { - throw std::ios_base::failure("Separator is missing at the end of the global map"); - } - - // Make sure that we got an unsigned tx - if (!tx) { - throw std::ios_base::failure("No unsigned transcation was provided"); - } - - // Read input data - unsigned int i = 0; - while (!s.empty() && i < tx->vin.size()) { - PSBTInput input; - s >> input; - inputs.push_back(input); - - // Make sure the non-witness utxo matches the outpoint - if (input.non_witness_utxo && input.non_witness_utxo->GetHash() != tx->vin[i].prevout.hash) { - throw std::ios_base::failure("Non-witness UTXO does not match outpoint hash"); - } - ++i; - } - // Make sure that the number of inputs matches the number of inputs in the transaction - if (inputs.size() != tx->vin.size()) { - throw std::ios_base::failure("Inputs provided does not match the number of inputs in transaction."); - } - - // Read output data - i = 0; - while (!s.empty() && i < tx->vout.size()) { - PSBTOutput output; - s >> output; - outputs.push_back(output); - ++i; - } - // Make sure that the number of outputs matches the number of outputs in the transaction - if (outputs.size() != tx->vout.size()) { - throw std::ios_base::failure("Outputs provided does not match the number of outputs in transaction."); - } - // Sanity check - if (!IsSane()) { - throw std::ios_base::failure("PSBT is not sane."); - } - } - - template <typename Stream> - PartiallySignedTransaction(deserialize_type, Stream& s) { - Unserialize(s); - } -}; - /** Produce a script signature using a generic signature creator. */ bool ProduceSignature(const SigningProvider& provider, const BaseSignatureCreator& creator, const CScript& scriptPubKey, SignatureData& sigdata); @@ -738,12 +222,6 @@ bool ProduceSignature(const SigningProvider& provider, const BaseSignatureCreato bool SignSignature(const SigningProvider &provider, const CScript& fromPubKey, CMutableTransaction& txTo, unsigned int nIn, const CAmount& amount, int nHashType); bool SignSignature(const SigningProvider &provider, const CTransaction& txFrom, CMutableTransaction& txTo, unsigned int nIn, int nHashType); -/** Checks whether a PSBTInput is already signed. */ -bool PSBTInputSigned(PSBTInput& input); - -/** Signs a PSBTInput, verifying that all provided data matches what is being signed. */ -bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, int sighash = SIGHASH_ALL); - /** Extract signature data from a transaction input, and insert it. */ SignatureData DataFromTransaction(const CMutableTransaction& tx, unsigned int nIn, const CTxOut& txout); void UpdateInput(CTxIn& input, const SignatureData& data); diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index 625362f446..cd0c36d802 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -112,6 +112,12 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test) BOOST_CHECK_EQUAL(block_filter.GetFilterType(), block_filter2.GetFilterType()); BOOST_CHECK_EQUAL(block_filter.GetBlockHash(), block_filter2.GetBlockHash()); BOOST_CHECK(block_filter.GetEncodedFilter() == block_filter2.GetEncodedFilter()); + + BlockFilter default_ctor_block_filter_1; + BlockFilter default_ctor_block_filter_2; + BOOST_CHECK_EQUAL(default_ctor_block_filter_1.GetFilterType(), default_ctor_block_filter_2.GetFilterType()); + BOOST_CHECK_EQUAL(default_ctor_block_filter_1.GetBlockHash(), default_ctor_block_filter_2.GetBlockHash()); + BOOST_CHECK(default_ctor_block_filter_1.GetEncodedFilter() == default_ctor_block_filter_2.GetEncodedFilter()); } BOOST_AUTO_TEST_CASE(blockfilters_json_test) diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 8da8cfc00c..ff2b8d4fc9 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -18,8 +18,8 @@ void CheckUnparsable(const std::string& prv, const std::string& pub) FlatSigningProvider keys_priv, keys_pub; auto parse_priv = Parse(prv, keys_priv); auto parse_pub = Parse(pub, keys_pub); - BOOST_CHECK(!parse_priv); - BOOST_CHECK(!parse_pub); + BOOST_CHECK_MESSAGE(!parse_priv, prv); + BOOST_CHECK_MESSAGE(!parse_pub, pub); } constexpr int DEFAULT = 0; @@ -28,6 +28,18 @@ constexpr int HARDENED = 2; // Derivation needs access to private keys constexpr int UNSOLVABLE = 4; // This descriptor is not expected to be solvable constexpr int SIGNABLE = 8; // We can sign with this descriptor (this is not true when actual BIP32 derivation is used, as that's not integrated in our signing code) +/** Compare two descriptors. If only one of them has a checksum, the checksum is ignored. */ +bool EqualDescriptor(std::string a, std::string b) +{ + bool a_check = (a.size() > 9 && a[a.size() - 9] == '#'); + bool b_check = (b.size() > 9 && b[b.size() - 9] == '#'); + if (a_check != b_check) { + if (a_check) a = a.substr(0, a.size() - 9); + if (b_check) b = b.substr(0, b.size() - 9); + } + return a == b; +} + std::string MaybeUseHInsteadOfApostrophy(std::string ret) { if (InsecureRandBool()) { @@ -35,6 +47,7 @@ std::string MaybeUseHInsteadOfApostrophy(std::string ret) auto it = ret.find("'"); if (it != std::string::npos) { ret[it] = 'h'; + if (ret.size() > 9 && ret[ret.size() - 9] == '#') ret = ret.substr(0, ret.size() - 9); // Changing apostrophe to h breaks the checksum } else { break; } @@ -63,16 +76,16 @@ void Check(const std::string& prv, const std::string& pub, int flags, const std: // Check that both versions serialize back to the public version. std::string pub1 = parse_priv->ToString(); std::string pub2 = parse_pub->ToString(); - BOOST_CHECK_EQUAL(pub, pub1); - BOOST_CHECK_EQUAL(pub, pub2); + BOOST_CHECK(EqualDescriptor(pub, pub1)); + BOOST_CHECK(EqualDescriptor(pub, pub2)); // Check that both can be serialized with private key back to the private version, but not without private key. std::string prv1; BOOST_CHECK(parse_priv->ToPrivateString(keys_priv, prv1)); - BOOST_CHECK_EQUAL(prv, prv1); + BOOST_CHECK(EqualDescriptor(prv, prv1)); BOOST_CHECK(!parse_priv->ToPrivateString(keys_pub, prv1)); BOOST_CHECK(parse_pub->ToPrivateString(keys_priv, prv1)); - BOOST_CHECK_EQUAL(prv, prv1); + BOOST_CHECK(EqualDescriptor(prv, prv1)); BOOST_CHECK(!parse_pub->ToPrivateString(keys_pub, prv1)); // Check whether IsRange on both returns the expected result @@ -210,6 +223,15 @@ BOOST_AUTO_TEST_CASE(descriptor_test) CheckUnparsable("wsh(sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "wsh(sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))"); // Cannot embed P2SH inside P2WSH CheckUnparsable("sh(sh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "sh(sh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))"); // Cannot embed P2SH inside P2SH CheckUnparsable("wsh(wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "wsh(wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))"); // Cannot embed P2WSH inside P2WSH + + // Checksums + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, {{0x8000006FUL,222},{0}}); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, {{0x8000006FUL,222},{0}}); + CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#"); // Empty checksum + CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfyq", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5tq"); // Too long checksum + CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxf", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5"); // Too short checksum + CheckUnparsable("sh(multi(3,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(3,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t"); // Error in payload + CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggssrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjq09x4t"); // Error in checksum } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp new file mode 100644 index 0000000000..079a09f8f9 --- /dev/null +++ b/src/test/flatfile_tests.cpp @@ -0,0 +1,123 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <flatfile.h> +#include <test/test_bitcoin.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(flatfile_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(flatfile_filename) +{ + auto data_dir = SetDataDir("flatfile_test"); + + FlatFilePos pos(456, 789); + + FlatFileSeq seq1(data_dir, "a", 16 * 1024); + BOOST_CHECK_EQUAL(seq1.FileName(pos), data_dir / "a00456.dat"); + + FlatFileSeq seq2(data_dir / "a", "b", 16 * 1024); + BOOST_CHECK_EQUAL(seq2.FileName(pos), data_dir / "a" / "b00456.dat"); +} + +BOOST_AUTO_TEST_CASE(flatfile_open) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 16 * 1024); + + std::string line1("A purely peer-to-peer version of electronic cash would allow online " + "payments to be sent directly from one party to another without going " + "through a financial institution."); + std::string line2("Digital signatures provide part of the solution, but the main benefits are " + "lost if a trusted third party is still required to prevent double-spending."); + + size_t pos1 = 0; + size_t pos2 = pos1 + GetSerializeSize(line1, CLIENT_VERSION); + + // Write first line to file. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos1)), SER_DISK, CLIENT_VERSION); + file << LIMITED_STRING(line1, 256); + } + + // Attempt to append to file opened in read-only mode. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos2), true), SER_DISK, CLIENT_VERSION); + BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), std::ios_base::failure); + } + + // Append second line to file. + { + CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + file << LIMITED_STRING(line2, 256); + } + + // Read text from file in read-only mode. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(0, pos1), true), SER_DISK, CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line1); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Read text from file with position offset. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(0, pos2)), SER_DISK, CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Ensure another file in the sequence has no data. + { + std::string text; + CAutoFile file(seq.Open(FlatFilePos(1, pos2)), SER_DISK, CLIENT_VERSION); + BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), std::ios_base::failure); + } +} + +BOOST_AUTO_TEST_CASE(flatfile_allocate) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 0), 1, out_of_space), 100); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 0))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 1, out_of_space), 0); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(FlatFilePos(0, 99), 2, out_of_space), 101); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 99))), 200); + BOOST_CHECK(!out_of_space); +} + +BOOST_AUTO_TEST_CASE(flatfile_flush) +{ + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + seq.Allocate(FlatFilePos(0, 0), 1, out_of_space); + + // Flush without finalize should not truncate file. + seq.Flush(FlatFilePos(0, 1)); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 100); + + // Flush with finalize should truncate file. + seq.Flush(FlatFilePos(0, 1), true); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(FlatFilePos(0, 1))), 1); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_bitcoin_fuzzy.cpp b/src/test/fuzz/deserialize.cpp index 859fba0bdc..859fba0bdc 100644 --- a/src/test/test_bitcoin_fuzzy.cpp +++ b/src/test/fuzz/deserialize.cpp diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index ad62a5faf0..8b03a7e46e 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -10,8 +10,6 @@ #include <vector> -const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; - void test_one_input(std::vector<uint8_t> buffer); #endif // BITCOIN_TEST_FUZZ_FUZZ_H diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp new file mode 100644 index 0000000000..2c0bfa360c --- /dev/null +++ b/src/test/fuzz/script_flags.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <script/interpreter.h> +#include <script/script.h> +#include <streams.h> +#include <version.h> + +#include <test/fuzz/fuzz.h> + +/** Flags that are not forbidden by an assert */ +static bool IsValidFlagCombination(unsigned flags); + +void test_one_input(std::vector<uint8_t> buffer) +{ + CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); + try { + int nVersion; + ds >> nVersion; + ds.SetVersion(nVersion); + } catch (const std::ios_base::failure&) { + return; + } + + try { + const CTransaction tx(deserialize, ds); + const PrecomputedTransactionData txdata(tx); + + unsigned int verify_flags; + ds >> verify_flags; + + if (!IsValidFlagCombination(verify_flags)) return; + + unsigned int fuzzed_flags; + ds >> fuzzed_flags; + + for (unsigned i = 0; i < tx.vin.size(); ++i) { + CTxOut prevout; + ds >> prevout; + + const TransactionSignatureChecker checker{&tx, i, prevout.nValue, txdata}; + + ScriptError serror; + const bool ret = VerifyScript(tx.vin.at(i).scriptSig, prevout.scriptPubKey, &tx.vin.at(i).scriptWitness, verify_flags, checker, &serror); + assert(ret == (serror == SCRIPT_ERR_OK)); + + // Verify that removing flags from a passing test or adding flags to a failing test does not change the result + if (ret) { + verify_flags &= ~fuzzed_flags; + } else { + verify_flags |= fuzzed_flags; + } + if (!IsValidFlagCombination(verify_flags)) return; + + ScriptError serror_fuzzed; + const bool ret_fuzzed = VerifyScript(tx.vin.at(i).scriptSig, prevout.scriptPubKey, &tx.vin.at(i).scriptWitness, verify_flags, checker, &serror_fuzzed); + assert(ret_fuzzed == (serror_fuzzed == SCRIPT_ERR_OK)); + + assert(ret_fuzzed == ret); + } + } catch (const std::ios_base::failure&) { + return; + } +} + +static bool IsValidFlagCombination(unsigned flags) +{ + if (flags & SCRIPT_VERIFY_CLEANSTACK && ~flags & (SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS)) return false; + if (flags & SCRIPT_VERIFY_WITNESS && ~flags & SCRIPT_VERIFY_P2SH) return false; + return true; +} diff --git a/src/test/main.cpp b/src/test/main.cpp new file mode 100644 index 0000000000..ff3f36b561 --- /dev/null +++ b/src/test/main.cpp @@ -0,0 +1,7 @@ +// Copyright (c) 2011-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#define BOOST_TEST_MODULE Bitcoin Core Test Suite + +#include <boost/test/unit_test.hpp> diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 0c3fb7c398..cdfd4db589 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -66,36 +66,36 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha { SetDataDir("tempdir"); const CChainParams& chainparams = Params(); - // Ideally we'd move all the RPC tests to the functional testing framework - // instead of unit tests, but for now we need these here. - - RegisterAllCoreRPCCommands(tableRPC); - ClearDatadirCache(); - - // We have to run a scheduler thread to prevent ActivateBestChain - // from blocking due to queue overrun. - threadGroup.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); - GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); - - mempool.setSanityCheck(1.0); - pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); - pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); - if (!LoadGenesisBlock(chainparams)) { - throw std::runtime_error("LoadGenesisBlock failed."); - } - { - CValidationState state; - if (!ActivateBestChain(state, chainparams)) { - throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", FormatStateMessage(state))); - } - } - nScriptCheckThreads = 3; - for (int i=0; i < nScriptCheckThreads-1; i++) - threadGroup.create_thread(&ThreadScriptCheck); - - g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); - g_connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. + // Ideally we'd move all the RPC tests to the functional testing framework + // instead of unit tests, but for now we need these here. + + RegisterAllCoreRPCCommands(tableRPC); + ClearDatadirCache(); + + // We have to run a scheduler thread to prevent ActivateBestChain + // from blocking due to queue overrun. + threadGroup.create_thread(std::bind(&CScheduler::serviceQueue, &scheduler)); + GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + + mempool.setSanityCheck(1.0); + pblocktree.reset(new CBlockTreeDB(1 << 20, true)); + pcoinsdbview.reset(new CCoinsViewDB(1 << 23, true)); + pcoinsTip.reset(new CCoinsViewCache(pcoinsdbview.get())); + if (!LoadGenesisBlock(chainparams)) { + throw std::runtime_error("LoadGenesisBlock failed."); + } + + CValidationState state; + if (!ActivateBestChain(state, chainparams)) { + throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", FormatStateMessage(state))); + } + + nScriptCheckThreads = 3; + for (int i = 0; i < nScriptCheckThreads - 1; i++) + threadGroup.create_thread(&ThreadScriptCheck); + + g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); + g_connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. } TestingSetup::~TestingSetup() diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index 4a06845683..38c6d85a8d 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -71,10 +71,6 @@ private: /** Testing setup that configures a complete environment. * Included are data directory, coins database, script check threads setup. */ -class CConnman; -class CNode; - -class PeerLogicValidation; struct TestingSetup : public BasicTestingSetup { boost::thread_group threadGroup; CScheduler scheduler; diff --git a/src/test/test_bitcoin_main.cpp b/src/test/test_bitcoin_main.cpp deleted file mode 100644 index 46b63b93b4..0000000000 --- a/src/test/test_bitcoin_main.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#define BOOST_TEST_MODULE Bitcoin Test Suite - -#include <banman.h> -#include <net.h> - -#include <memory> - -#include <boost/test/unit_test.hpp> - -std::unique_ptr<CConnman> g_connman; -std::unique_ptr<BanMan> g_banman; - -[[noreturn]] void Shutdown(void* parg) -{ - std::exit(EXIT_SUCCESS); -} - -[[noreturn]] void StartShutdown() -{ - std::exit(EXIT_SUCCESS); -} - -bool ShutdownRequested() -{ - return false; -} diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 0301901bf0..d667c26c3c 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -69,7 +69,13 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) } } - txindex.Stop(); // Stop thread before calling destructor + // shutdown sequence (c.f. Shutdown() in init.cpp) + txindex.Stop(); + + threadGroup.interrupt_all(); + threadGroup.join_all(); + + // Rest of shutdown sequence and destructors happen in ~TestingSetup() } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index c467f27836..4d04aae7e9 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -102,7 +102,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // should fail. // Capture this interaction with the upgraded_nop argument: set it when evaluating // any script flag that is implemented as an upgraded NOP code. -static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) +static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { PrecomputedTransactionData txdata(tx); // If we add many more flags, this loop can get too expensive, but we can @@ -219,11 +219,10 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) CBlock block; block = CreateAndProcessBlock({spend_tx}, p2pk_scriptPubKey); + LOCK(cs_main); BOOST_CHECK(chainActive.Tip()->GetBlockHash() == block.GetHash()); BOOST_CHECK(pcoinsTip->GetBestBlock() == block.GetHash()); - LOCK(cs_main); - // Test P2SH: construct a transaction that is valid without P2SH, and // then test validity with P2SH. { diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 71b6ec7425..e17ae7103b 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -169,11 +169,6 @@ BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) BOOST_CHECK_EQUAL(FormatISO8601Date(1317425777), "2011-09-30"); } -BOOST_AUTO_TEST_CASE(util_FormatISO8601Time) -{ - BOOST_CHECK_EQUAL(FormatISO8601Time(1317425777), "23:36:17Z"); -} - struct TestArgsManager : public ArgsManager { TestArgsManager() { m_network_only_args.clear(); } @@ -185,9 +180,10 @@ struct TestArgsManager : public ArgsManager { LOCK(cs_args); m_config_args.clear(); + m_config_sections.clear(); } std::string error; - BOOST_REQUIRE(ReadConfigStream(streamConfig, error)); + BOOST_REQUIRE(ReadConfigStream(streamConfig, "", error)); } void SetNetworkOnlyArg(const std::string arg) { diff --git a/src/test/main_tests.cpp b/src/test/validation_tests.cpp index 5b3f2bc578..8d06ecd3a9 100644 --- a/src/test/main_tests.cpp +++ b/src/test/validation_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2014-2018 The Bitcoin Core developers +// Copyright (c) 2014-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <validation.h> #include <net.h> +#include <validation.h> #include <test/test_bitcoin.h> #include <boost/signals2/signal.hpp> #include <boost/test/unit_test.hpp> -BOOST_FIXTURE_TEST_SUITE(main_tests, TestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_tests, TestingSetup) static void TestBlockSubsidyHalvings(const Consensus::Params& consensusParams) { diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 68f47d5cce..ca556bdc7b 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -764,7 +764,7 @@ std::vector<CTxMemPool::indexed_transaction_set::const_iterator> CTxMemPool::Get return iters; } -void CTxMemPool::queryHashes(std::vector<uint256>& vtxid) +void CTxMemPool::queryHashes(std::vector<uint256>& vtxid) const { LOCK(cs); auto iters = GetSortedDepthAndScore(); diff --git a/src/txmempool.h b/src/txmempool.h index f7afaec8fc..a8a0f7fa45 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -184,7 +184,7 @@ private: const LockPoints& lp; }; -// extracts a transaction hash from CTxMempoolEntry or CTransactionRef +// extracts a transaction hash from CTxMemPoolEntry or CTransactionRef struct mempoolentry_txid { typedef uint256 result_type; @@ -588,7 +588,7 @@ public: void clear(); void _clear() EXCLUSIVE_LOCKS_REQUIRED(cs); //lock free bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb); - void queryHashes(std::vector<uint256>& vtxid); + void queryHashes(std::vector<uint256>& vtxid) const; bool isSpent(const COutPoint& outpoint) const; unsigned int GetTransactionsUpdated() const; void AddTransactionsUpdated(unsigned int n); diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp index 947d7e2308..16ab24686b 100644 --- a/src/ui_interface.cpp +++ b/src/ui_interface.cpp @@ -52,7 +52,7 @@ void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_s void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } void CClientUIInterface::NotifyAlertChanged() { return g_ui_signals.NotifyAlertChanged(); } -void CClientUIInterface::LoadWallet(std::shared_ptr<CWallet> wallet) { return g_ui_signals.LoadWallet(wallet); } +void CClientUIInterface::LoadWallet(std::unique_ptr<interfaces::Wallet>& wallet) { return g_ui_signals.LoadWallet(wallet); } void CClientUIInterface::ShowProgress(const std::string& title, int nProgress, bool resume_possible) { return g_ui_signals.ShowProgress(title, nProgress, resume_possible); } void CClientUIInterface::NotifyBlockTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyBlockTip(b, i); } void CClientUIInterface::NotifyHeaderTip(bool b, const CBlockIndex* i) { return g_ui_signals.NotifyHeaderTip(b, i); } diff --git a/src/ui_interface.h b/src/ui_interface.h index fe466b3ca4..f1aebce3bb 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -11,7 +11,6 @@ #include <stdint.h> #include <string> -class CWallet; class CBlockIndex; namespace boost { namespace signals2 { @@ -19,6 +18,10 @@ class connection; } } // namespace boost +namespace interfaces { +class Wallet; +} // namespace interfaces + /** General change type (added, updated, removed). */ enum ChangeType { @@ -102,7 +105,7 @@ public: ADD_SIGNALS_DECL_WRAPPER(NotifyAlertChanged, void, ); /** A wallet has been loaded. */ - ADD_SIGNALS_DECL_WRAPPER(LoadWallet, void, std::shared_ptr<CWallet> wallet); + ADD_SIGNALS_DECL_WRAPPER(LoadWallet, void, std::unique_ptr<interfaces::Wallet>& wallet); /** * Show progress e.g. for verifychain. diff --git a/src/util/bip32.cpp b/src/util/bip32.cpp new file mode 100644 index 0000000000..6f176dd5ec --- /dev/null +++ b/src/util/bip32.cpp @@ -0,0 +1,66 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <sstream> +#include <stdio.h> +#include <tinyformat.h> +#include <util/bip32.h> +#include <util/strencodings.h> + + +bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath) +{ + std::stringstream ss(keypath_str); + std::string item; + bool first = true; + while (std::getline(ss, item, '/')) { + if (item.compare("m") == 0) { + if (first) { + first = false; + continue; + } + return false; + } + // Finds whether it is hardened + uint32_t path = 0; + size_t pos = item.find("'"); + if (pos != std::string::npos) { + // The hardened tick can only be in the last index of the string + if (pos != item.size() - 1) { + return false; + } + path |= 0x80000000; + item = item.substr(0, item.size() - 1); // Drop the last character which is the hardened tick + } + + // Ensure this is only numbers + if (item.find_first_not_of( "0123456789" ) != std::string::npos) { + return false; + } + uint32_t number; + if (!ParseUInt32(item, &number)) { + return false; + } + path |= number; + + keypath.push_back(path); + first = false; + } + return true; +} + +std::string FormatHDKeypath(const std::vector<uint32_t>& path) +{ + std::string ret; + for (auto i : path) { + ret += strprintf("/%i", (i << 1) >> 1); + if (i >> 31) ret += '\''; + } + return ret; +} + +std::string WriteHDKeypath(const std::vector<uint32_t>& keypath) +{ + return "m" + FormatHDKeypath(keypath); +} diff --git a/src/util/bip32.h b/src/util/bip32.h new file mode 100644 index 0000000000..7e58b79f38 --- /dev/null +++ b/src/util/bip32.h @@ -0,0 +1,19 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_BIP32_H +#define BITCOIN_UTIL_BIP32_H + +#include <attributes.h> +#include <string> +#include <vector> + +/** Parse an HD keypaths like "m/7/0'/2000". */ +NODISCARD bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath); + +/** Write HD keypaths as strings */ +std::string WriteHDKeypath(const std::vector<uint32_t>& keypath); +std::string FormatHDKeypath(const std::vector<uint32_t>& path); + +#endif // BITCOIN_UTIL_BIP32_H diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index fedeeac39b..0acbb4f117 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -141,7 +141,7 @@ std::string EncodeBase64(const std::string& str) return EncodeBase64((const unsigned char*)str.c_str(), str.size()); } -std::vector<unsigned char> DecodeBase64(const char* p, bool* pfInvalid) +std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) { static const int decode64_table[256] = { @@ -183,14 +183,14 @@ std::vector<unsigned char> DecodeBase64(const char* p, bool* pfInvalid) ++p; } valid = valid && (p - e) % 4 == 0 && p - q < 4; - if (pfInvalid) *pfInvalid = !valid; + if (pf_invalid) *pf_invalid = !valid; return ret; } -std::string DecodeBase64(const std::string& str) +std::string DecodeBase64(const std::string& str, bool* pf_invalid) { - std::vector<unsigned char> vchRet = DecodeBase64(str.c_str()); + std::vector<unsigned char> vchRet = DecodeBase64(str.c_str(), pf_invalid); return std::string((const char*)vchRet.data(), vchRet.size()); } @@ -210,7 +210,7 @@ std::string EncodeBase32(const std::string& str) return EncodeBase32((const unsigned char*)str.c_str(), str.size()); } -std::vector<unsigned char> DecodeBase32(const char* p, bool* pfInvalid) +std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid) { static const int decode32_table[256] = { @@ -252,14 +252,14 @@ std::vector<unsigned char> DecodeBase32(const char* p, bool* pfInvalid) ++p; } valid = valid && (p - e) % 8 == 0 && p - q < 8; - if (pfInvalid) *pfInvalid = !valid; + if (pf_invalid) *pf_invalid = !valid; return ret; } -std::string DecodeBase32(const std::string& str) +std::string DecodeBase32(const std::string& str, bool* pf_invalid) { - std::vector<unsigned char> vchRet = DecodeBase32(str.c_str()); + std::vector<unsigned char> vchRet = DecodeBase32(str.c_str(), pf_invalid); return std::string((const char*)vchRet.data(), vchRet.size()); } @@ -546,47 +546,6 @@ bool ParseFixedPoint(const std::string &val, int decimals, int64_t *amount_out) return true; } -bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath) -{ - std::stringstream ss(keypath_str); - std::string item; - bool first = true; - while (std::getline(ss, item, '/')) { - if (item.compare("m") == 0) { - if (first) { - first = false; - continue; - } - return false; - } - // Finds whether it is hardened - uint32_t path = 0; - size_t pos = item.find("'"); - if (pos != std::string::npos) { - // The hardened tick can only be in the last index of the string - if (pos != item.size() - 1) { - return false; - } - path |= 0x80000000; - item = item.substr(0, item.size() - 1); // Drop the last character which is the hardened tick - } - - // Ensure this is only numbers - if (item.find_first_not_of( "0123456789" ) != std::string::npos) { - return false; - } - uint32_t number; - if (!ParseUInt32(item, &number)) { - return false; - } - path |= number; - - keypath.push_back(path); - first = false; - } - return true; -} - void Downcase(std::string& str) { std::transform(str.begin(), str.end(), str.begin(), [](char c){return ToLower(c);}); diff --git a/src/util/strencodings.h b/src/util/strencodings.h index e392055f27..cf77044094 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -44,12 +44,12 @@ bool IsHex(const std::string& str); * Return true if the string is a hex number, optionally prefixed with "0x" */ bool IsHexNumber(const std::string& str); -std::vector<unsigned char> DecodeBase64(const char* p, bool* pfInvalid = nullptr); -std::string DecodeBase64(const std::string& str); +std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid = nullptr); +std::string DecodeBase64(const std::string& str, bool* pf_invalid = nullptr); std::string EncodeBase64(const unsigned char* pch, size_t len); std::string EncodeBase64(const std::string& str); -std::vector<unsigned char> DecodeBase32(const char* p, bool* pfInvalid = nullptr); -std::string DecodeBase32(const std::string& str); +std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid = nullptr); +std::string DecodeBase32(const std::string& str, bool* pf_invalid = nullptr); std::string EncodeBase32(const unsigned char* pch, size_t len); std::string EncodeBase32(const std::string& str); @@ -197,9 +197,6 @@ bool ConvertBits(const O& outfn, I it, I end) { return true; } -/** Parse an HD keypaths like "m/7/0'/2000". */ -NODISCARD bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath); - /** * Converts the given character to its lowercase equivalent. * This function is locale independent. It only converts uppercase diff --git a/src/util/system.cpp b/src/util/system.cpp index bb9fcab59e..9594dd81bf 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -74,7 +74,6 @@ const int64_t nStartupTime = GetTime(); const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; -const char * const BITCOIN_PID_FILENAME = "bitcoind.pid"; ArgsManager gArgs; @@ -136,6 +135,14 @@ bool DirIsWritable(const fs::path& directory) return true; } +bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes) +{ + constexpr uint64_t min_disk_space = 52428800; // 50 MiB + + uint64_t free_bytes_available = fs::space(dir).available; + return free_bytes_available >= min_disk_space + additional_bytes; +} + /** * Interpret a string argument as a boolean. * @@ -354,8 +361,7 @@ const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const return unsuitables; } - -const std::set<std::string> ArgsManager::GetUnrecognizedSections() const +const std::list<SectionInfo> ArgsManager::GetUnrecognizedSections() const { // Section names to be recognized in the config file. static const std::set<std::string> available_sections{ @@ -363,14 +369,11 @@ const std::set<std::string> ArgsManager::GetUnrecognizedSections() const CBaseChainParams::TESTNET, CBaseChainParams::MAIN }; - std::set<std::string> diff; LOCK(cs_args); - std::set_difference( - m_config_sections.begin(), m_config_sections.end(), - available_sections.begin(), available_sections.end(), - std::inserter(diff, diff.end())); - return diff; + std::list<SectionInfo> unrecognized = m_config_sections; + unrecognized.remove_if([](const SectionInfo& appeared){ return available_sections.find(appeared.m_name) != available_sections.end(); }); + return unrecognized; } void ArgsManager::SelectConfigNetwork(const std::string& network) @@ -635,6 +638,12 @@ bool HelpRequested(const ArgsManager& args) return args.IsArgSet("-?") || args.IsArgSet("-h") || args.IsArgSet("-help") || args.IsArgSet("-help-debug"); } +void SetupHelpOptions(ArgsManager& args) +{ + args.AddArg("-?", "Print this help message and exit", false, OptionsCategory::OPTIONS); + args.AddHiddenArgs({"-h", "-help"}); +} + static const int screenWidth = 79; static const int optIndent = 2; static const int msgIndent = 7; @@ -788,7 +797,7 @@ static std::string TrimString(const std::string& str, const std::string& pattern return str.substr(front, end - front + 1); } -static bool GetConfigOptions(std::istream& stream, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::set<std::string>& sections) +static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) { std::string str, prefix; std::string::size_type pos; @@ -804,7 +813,7 @@ static bool GetConfigOptions(std::istream& stream, std::string& error, std::vect if (!str.empty()) { if (*str.begin() == '[' && *str.rbegin() == ']') { const std::string section = str.substr(1, str.size() - 2); - sections.insert(section); + sections.emplace_back(SectionInfo{section, filepath, linenr}); prefix = section + '.'; } else if (*str.begin() == '-') { error = strprintf("parse error on line %i: %s, options in configuration file must be specified without leading -", linenr, str); @@ -817,8 +826,8 @@ static bool GetConfigOptions(std::istream& stream, std::string& error, std::vect return false; } options.emplace_back(name, value); - if ((pos = name.rfind('.')) != std::string::npos) { - sections.insert(name.substr(0, pos)); + if ((pos = name.rfind('.')) != std::string::npos && prefix.length() <= pos) { + sections.emplace_back(SectionInfo{name.substr(0, pos), filepath, linenr}); } } else { error = strprintf("parse error on line %i: %s", linenr, str); @@ -833,12 +842,11 @@ static bool GetConfigOptions(std::istream& stream, std::string& error, std::vect return true; } -bool ArgsManager::ReadConfigStream(std::istream& stream, std::string& error, bool ignore_invalid_keys) +bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys) { LOCK(cs_args); std::vector<std::pair<std::string, std::string>> options; - m_config_sections.clear(); - if (!GetConfigOptions(stream, error, options, m_config_sections)) { + if (!GetConfigOptions(stream, filepath, error, options, m_config_sections)) { return false; } for (const std::pair<std::string, std::string>& option : options) { @@ -869,6 +877,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) { LOCK(cs_args); m_config_args.clear(); + m_config_sections.clear(); } const std::string confPath = GetArg("-conf", BITCOIN_CONF_FILENAME); @@ -876,7 +885,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) // ok to not have a config file if (stream.good()) { - if (!ReadConfigStream(stream, error, ignore_invalid_keys)) { + if (!ReadConfigStream(stream, confPath, error, ignore_invalid_keys)) { return false; } // if there is an -includeconf in the override args, but it is empty, that means the user @@ -907,7 +916,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) for (const std::string& to_include : includeconf) { fsbridge::ifstream include_config(GetConfigFile(to_include)); if (include_config.good()) { - if (!ReadConfigStream(include_config, error, ignore_invalid_keys)) { + if (!ReadConfigStream(include_config, to_include, error, ignore_invalid_keys)) { return false; } LogPrintf("Included configuration file %s\n", to_include.c_str()); @@ -959,23 +968,6 @@ std::string ArgsManager::GetChainName() const return CBaseChainParams::MAIN; } -#ifndef WIN32 -fs::path GetPidFile() -{ - return AbsPathForConfigVal(fs::path(gArgs.GetArg("-pid", BITCOIN_PID_FILENAME))); -} - -void CreatePidFile(const fs::path &path, pid_t pid) -{ - FILE* file = fsbridge::fopen(path, "w"); - if (file) - { - fprintf(file, "%d\n", pid); - fclose(file); - } -} -#endif - bool RenameOver(fs::path src, fs::path dest) { #ifdef WIN32 diff --git a/src/util/system.h b/src/util/system.h index 17723d427d..54eb88e261 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -16,6 +16,7 @@ #include <attributes.h> #include <compat.h> +#include <compat/assumptions.h> #include <fs.h> #include <logging.h> #include <sync.h> @@ -39,7 +40,6 @@ int64_t GetStartupTime(); extern const char * const BITCOIN_CONF_FILENAME; -extern const char * const BITCOIN_PID_FILENAME; /** Translate a message to the native language of the user. */ const extern std::function<std::string(const char*)> G_TRANSLATION_FUN; @@ -72,6 +72,7 @@ bool RenameOver(fs::path src, fs::path dest); bool LockDirectory(const fs::path& directory, const std::string lockfile_name, bool probe_only=false); void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name); bool DirIsWritable(const fs::path& directory); +bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0); /** Release all directory locks. This is used for unit testing only, at runtime * the global destructor will take care of the locks. @@ -85,10 +86,6 @@ const fs::path &GetBlocksDir(); const fs::path &GetDataDir(bool fNetSpecific = true); void ClearDatadirCache(); fs::path GetConfigFile(const std::string& confPath); -#ifndef WIN32 -fs::path GetPidFile(); -void CreatePidFile(const fs::path &path, pid_t pid); -#endif #ifdef WIN32 fs::path GetSpecialFolderPath(int nFolder, bool fCreate = true); #endif @@ -131,6 +128,13 @@ enum class OptionsCategory { HIDDEN // Always the last option to avoid printing these in the help }; +struct SectionInfo +{ + std::string m_name; + std::string m_file; + int m_line; +}; + class ArgsManager { protected: @@ -151,9 +155,9 @@ protected: std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); - std::set<std::string> m_config_sections GUARDED_BY(cs_args); + std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); - NODISCARD bool ReadConfigStream(std::istream& stream, std::string& error, bool ignore_invalid_keys = false); + NODISCARD bool ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys = false); public: ArgsManager(); @@ -177,7 +181,7 @@ public: /** * Log warnings for unrecognized section names in the config file. */ - const std::set<std::string> GetUnrecognizedSections() const; + const std::list<SectionInfo> GetUnrecognizedSections() const; /** * Return a vector of strings of the given argument @@ -295,6 +299,9 @@ extern ArgsManager gArgs; */ bool HelpRequested(const ArgsManager& args); +/** Add help options to the args manager */ +void SetupHelpOptions(ArgsManager& args); + /** * Format a string to be used as group of options in help messages * diff --git a/src/util/time.cpp b/src/util/time.cpp index 83a7937d8f..c0ede98701 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -97,14 +97,3 @@ std::string FormatISO8601Date(int64_t nTime) { #endif return strprintf("%04i-%02i-%02i", ts.tm_year + 1900, ts.tm_mon + 1, ts.tm_mday); } - -std::string FormatISO8601Time(int64_t nTime) { - struct tm ts; - time_t time_val = nTime; -#ifdef _MSC_VER - gmtime_s(&ts, &time_val); -#else - gmtime_r(&time_val, &ts); -#endif - return strprintf("%02i:%02i:%02iZ", ts.tm_hour, ts.tm_min, ts.tm_sec); -} diff --git a/src/util/time.h b/src/util/time.h index f2e2747434..68de1c156e 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -33,6 +33,5 @@ void MilliSleep(int64_t n); */ std::string FormatISO8601DateTime(int64_t nTime); std::string FormatISO8601Date(int64_t nTime); -std::string FormatISO8601Time(int64_t nTime); #endif // BITCOIN_UTIL_TIME_H diff --git a/src/validation.cpp b/src/validation.cpp index de9c0d96db..a9b5bcf4a4 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -15,6 +15,7 @@ #include <consensus/tx_verify.h> #include <consensus/validation.h> #include <cuckoocache.h> +#include <flatfile.h> #include <hash.h> #include <index/txindex.h> #include <policy/fees.h> @@ -152,7 +153,7 @@ private: public: CChain chainActive; - BlockMap mapBlockIndex; + BlockMap mapBlockIndex GUARDED_BY(cs_main); std::multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; CBlockIndex *pindexBestInvalid = nullptr; @@ -165,7 +166,7 @@ public: * that it doesn't descend from an invalid block, and then add it to mapBlockIndex. */ bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); @@ -177,7 +178,7 @@ public: // Manual block validity manipulation: bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); - bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex); void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool ReplayBlocks(const CChainParams& params, CCoinsView* view); @@ -204,10 +205,12 @@ private: void InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - + void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! Mark a block as not having block data + void EraseBlockData(CBlockIndex* index) EXCLUSIVE_LOCKS_REQUIRED(cs_main); } g_chainstate; /** @@ -317,7 +320,9 @@ static bool FlushStateToDisk(const CChainParams& chainParams, CValidationState & static void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight); static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight); bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck> *pvChecks = nullptr); -static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); bool CheckFinalTx(const CTransaction &tx, int flags) { @@ -1042,7 +1047,7 @@ bool GetTransaction(const uint256& hash, CTransactionRef& txOut, const Consensus // CBlock and CBlockIndex // -static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart) +static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); @@ -1063,7 +1068,7 @@ static bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMes return true; } -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); @@ -1089,7 +1094,7 @@ bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus: bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) { - CDiskBlockPos blockPos; + FlatFilePos blockPos; { LOCK(cs_main); blockPos = pindex->GetBlockPos(); @@ -1103,9 +1108,9 @@ bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus return true; } -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start) +bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) { - CDiskBlockPos hpos = pos; + FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) { @@ -1140,7 +1145,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start) { - CDiskBlockPos block_pos; + FlatFilePos block_pos; { LOCK(cs_main); block_pos = pindex->GetBlockPos(); @@ -1452,7 +1457,7 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi namespace { -bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); @@ -1481,7 +1486,7 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint static bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex *pindex) { - CDiskBlockPos pos = pindex->GetUndoPos(); + FlatFilePos pos = pindex->GetUndoPos(); if (pos.IsNull()) { return error("%s: no undo data available", __func__); } @@ -1627,37 +1632,24 @@ void static FlushBlockFile(bool fFinalize = false) { LOCK(cs_LastBlockFile); - CDiskBlockPos posOld(nLastBlockFile, 0); - bool status = true; - - FILE *fileOld = OpenBlockFile(posOld); - if (fileOld) { - if (fFinalize) - status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize); - status &= FileCommit(fileOld); - fclose(fileOld); - } - - fileOld = OpenUndoFile(posOld); - if (fileOld) { - if (fFinalize) - status &= TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize); - status &= FileCommit(fileOld); - fclose(fileOld); - } + FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + FlatFilePos undo_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nUndoSize); + bool status = true; + status &= BlockFileSeq().Flush(block_pos_old, fFinalize); + status &= UndoFileSeq().Flush(undo_pos_old, fFinalize); if (!status) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); } } -static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize); +static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) { // Write undo information to disk if (pindex->GetUndoPos().IsNull()) { - CDiskBlockPos _pos; + FlatFilePos _pos; if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) return error("ConnectBlock(): FindUndoPos failed"); if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) @@ -2134,8 +2126,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Depend on nMinDiskSpace to ensure we can write block index - if (!CheckDiskSpace(0, true)) - return state.Error("out of disk space"); + if (!CheckDiskSpace(GetBlocksDir())) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } // First make sure all block and undo data is flushed to disk. FlushBlockFile(); // Then update all block file information (which may refer to block and undo files). @@ -2168,8 +2161,9 @@ bool static FlushStateToDisk(const CChainParams& chainparams, CValidationState & // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(48 * 2 * 2 * pcoinsTip->GetCacheSize())) - return state.Error("out of disk space"); + if (!CheckDiskSpace(GetDataDir(), 48 * 2 * 2 * pcoinsTip->GetCacheSize())) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return AbortNode(state, "Failed to write to coin database"); @@ -2259,12 +2253,6 @@ void static UpdateTip(const CBlockIndex *pindexNew, const CChainParams& chainPar } if (nUpgraded > 0) AppendWarning(warningMessages, strprintf(_("%d of last 100 blocks have unexpected version"), nUpgraded)); - if (nUpgraded > 100/2) - { - std::string strWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); - // notify GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - DoWarning(strWarning); - } } LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)", __func__, /* Continued */ pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, @@ -2640,6 +2628,14 @@ static void NotifyHeaderTip() LOCKS_EXCLUDED(cs_main) { } } +static void LimitValidationInterfaceQueue() { + AssertLockNotHeld(cs_main); + + if (GetMainSignals().CallbacksPending() > 10) { + SyncWithValidationInterfaceQueue(); + } +} + /** * Make the best chain active, in multiple steps. The result is either failure * or an activated best chain. pblock is either nullptr or a pointer to a block @@ -2668,15 +2664,13 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& do { boost::this_thread::interruption_point(); - if (GetMainSignals().CallbacksPending() > 10) { - // Block until the validation queue drains. This should largely - // never happen in normal operation, however may happen during - // reindex, causing memory blowup if we run too far ahead. - // Note that if a validationinterface callback ends up calling - // ActivateBestChain this may lead to a deadlock! We should - // probably have a DEBUG_LOCKORDER test for this in the future. - SyncWithValidationInterfaceQueue(); - } + // Block until the validation queue drains. This should largely + // never happen in normal operation, however may happen during + // reindex, causing memory blowup if we run too far ahead. + // Note that if a validationinterface callback ends up calling + // ActivateBestChain this may lead to a deadlock! We should + // probably have a DEBUG_LOCKORDER test for this in the future. + LimitValidationInterfaceQueue(); { LOCK(cs_main); @@ -2787,64 +2781,85 @@ bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIn bool CChainState::InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) { - AssertLockHeld(cs_main); + CBlockIndex* to_mark_failed = pindex; + bool pindex_was_in_chain = false; + int disconnected = 0; - // We first disconnect backwards and then mark the blocks as invalid. - // This prevents a case where pruned nodes may fail to invalidateblock - // and be left unable to start as they have no tip candidates (as there - // are no blocks that meet the "have data and are not invalid per - // nStatus" criteria for inclusion in setBlockIndexCandidates). + // Disconnect (descendants of) pindex, and mark them invalid. + while (true) { + if (ShutdownRequested()) break; - bool pindex_was_in_chain = false; - CBlockIndex *invalid_walk_tip = chainActive.Tip(); + // Make sure the queue of validation callbacks doesn't grow unboundedly. + LimitValidationInterfaceQueue(); - DisconnectedBlockTransactions disconnectpool; - while (chainActive.Contains(pindex)) { + LOCK(cs_main); + if (!chainActive.Contains(pindex)) break; pindex_was_in_chain = true; + CBlockIndex *invalid_walk_tip = chainActive.Tip(); + // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. - if (!DisconnectTip(state, chainparams, &disconnectpool)) { - // It's probably hopeless to try to make the mempool consistent - // here if DisconnectTip failed, but we can try. - UpdateMempoolForReorg(disconnectpool, false); - return false; - } - } - - // Now mark the blocks we just disconnected as descendants invalid - // (note this may not be all descendants). - while (pindex_was_in_chain && invalid_walk_tip != pindex) { - invalid_walk_tip->nStatus |= BLOCK_FAILED_CHILD; + DisconnectedBlockTransactions disconnectpool; + bool ret = DisconnectTip(state, chainparams, &disconnectpool); + // DisconnectTip will add transactions to disconnectpool. + // Adjust the mempool to be consistent with the new tip, adding + // transactions back to the mempool if disconnecting was succesful, + // and we're not doing a very deep invalidation (in which case + // keeping the mempool up to date is probably futile anyway). + UpdateMempoolForReorg(disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); + if (!ret) return false; + assert(invalid_walk_tip->pprev == chainActive.Tip()); + + // We immediately mark the disconnected blocks as invalid. + // This prevents a case where pruned nodes may fail to invalidateblock + // and be left unable to start as they have no tip candidates (as there + // are no blocks that meet the "have data and are not invalid per + // nStatus" criteria for inclusion in setBlockIndexCandidates). + invalid_walk_tip->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(invalid_walk_tip); setBlockIndexCandidates.erase(invalid_walk_tip); - invalid_walk_tip = invalid_walk_tip->pprev; + setBlockIndexCandidates.insert(invalid_walk_tip->pprev); + if (invalid_walk_tip->pprev == to_mark_failed && (to_mark_failed->nStatus & BLOCK_FAILED_VALID)) { + // We only want to mark the last disconnected block as BLOCK_FAILED_VALID; its children + // need to be BLOCK_FAILED_CHILD instead. + to_mark_failed->nStatus = (to_mark_failed->nStatus ^ BLOCK_FAILED_VALID) | BLOCK_FAILED_CHILD; + setDirtyBlockIndex.insert(to_mark_failed); + } + + // Track the last disconnected block, so we can correct its BLOCK_FAILED_CHILD status in future + // iterations, or, if it's the last one, call InvalidChainFound on it. + to_mark_failed = invalid_walk_tip; } - // Mark the block itself as invalid. - pindex->nStatus |= BLOCK_FAILED_VALID; - setDirtyBlockIndex.insert(pindex); - setBlockIndexCandidates.erase(pindex); - m_failed_blocks.insert(pindex); + { + LOCK(cs_main); + if (chainActive.Contains(to_mark_failed)) { + // If the to-be-marked invalid block is in the active chain, something is interfering and we can't proceed. + return false; + } - // DisconnectTip will add transactions to disconnectpool; try to add these - // back to the mempool. - UpdateMempoolForReorg(disconnectpool, true); + // Mark pindex (or the last disconnected block) as invalid, even when it never was in the main chain + to_mark_failed->nStatus |= BLOCK_FAILED_VALID; + setDirtyBlockIndex.insert(to_mark_failed); + setBlockIndexCandidates.erase(to_mark_failed); + m_failed_blocks.insert(to_mark_failed); - // The resulting new best tip may not be in setBlockIndexCandidates anymore, so - // add it again. - BlockMap::iterator it = mapBlockIndex.begin(); - while (it != mapBlockIndex.end()) { - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { - setBlockIndexCandidates.insert(it->second); + // The resulting new best tip may not be in setBlockIndexCandidates anymore, so + // add it again. + BlockMap::iterator it = mapBlockIndex.begin(); + while (it != mapBlockIndex.end()) { + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->HaveTxsDownloaded() && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { + setBlockIndexCandidates.insert(it->second); + } + it++; } - it++; - } - InvalidChainFound(pindex); + InvalidChainFound(to_mark_failed); + } // Only notify about a new block tip if the active chain was modified. if (pindex_was_in_chain) { - uiInterface.NotifyBlockTip(IsInitialBlockDownload(), pindex->pprev); + uiInterface.NotifyBlockTip(IsInitialBlockDownload(), to_mark_failed->pprev); } return true; } @@ -2928,7 +2943,7 @@ CBlockIndex* CChainState::AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) +void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2974,7 +2989,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi } } -static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) +static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown = false) { LOCK(cs_LastBlockFile); @@ -3009,21 +3024,13 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int vinfoBlockFile[nFile].nSize += nAddSize; if (!fKnown) { - unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos, true)) { - FILE *file = OpenBlockFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return error("out of disk space"); + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is low!", _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } } @@ -3031,32 +3038,23 @@ static bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int return true; } -static bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigned int nAddSize) +static bool FindUndoPos(CValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) { pos.nFile = nFile; LOCK(cs_LastBlockFile); - unsigned int nNewSize; pos.nPos = vinfoBlockFile[nFile].nUndoSize; - nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; + vinfoBlockFile[nFile].nUndoSize += nAddSize; setDirtyFileInfo.insert(nFile); - unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) - fCheckForPruning = true; - if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos, true)) { - FILE *file = OpenUndoFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } - else - return state.Error("out of disk space"); + bool out_of_space; + size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is low!", _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } return true; @@ -3441,26 +3439,26 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, CValidatio } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -static CDiskBlockPos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const CDiskBlockPos* dbp) { +static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); - CDiskBlockPos blockPos; + FlatFilePos blockPos; if (dbp != nullptr) blockPos = *dbp; if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != nullptr)) { error("%s: FindBlockPos failed", __func__); - return CDiskBlockPos(); + return FlatFilePos(); } if (dbp == nullptr) { if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { AbortNode("Failed to write block"); - return CDiskBlockPos(); + return FlatFilePos(); } } return blockPos; } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) +bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; @@ -3522,7 +3520,7 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, CVali // Write block to history file if (fNewBlock) *fNewBlock = true; try { - CDiskBlockPos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); + FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, chainparams, dbp); if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -3653,9 +3651,9 @@ void PruneOneBlockFile(const int fileNumber) void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) { for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { - CDiskBlockPos pos(*it, 0); - fs::remove(GetBlockPosFilename(pos, "blk")); - fs::remove(GetBlockPosFilename(pos, "rev")); + FlatFilePos pos(*it, 0); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } @@ -3763,52 +3761,28 @@ static void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfte nLastBlockWeCanPrune, count); } -bool CheckDiskSpace(uint64_t nAdditionalBytes, bool blocks_dir) +static FlatFileSeq BlockFileSeq() { - uint64_t nFreeBytesAvailable = fs::space(blocks_dir ? GetBlocksDir() : GetDataDir()).available; - - // Check for nMinDiskSpace bytes (currently 50MB) - if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) - return AbortNode("Disk space is low!", _("Error: Disk space is low!")); - - return true; + return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE); } -static FILE* OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, bool fReadOnly) +static FlatFileSeq UndoFileSeq() { - if (pos.IsNull()) - return nullptr; - fs::path path = GetBlockPosFilename(pos, prefix); - fs::create_directories(path.parent_path()); - FILE* file = fsbridge::fopen(path, fReadOnly ? "rb": "rb+"); - if (!file && !fReadOnly) - file = fsbridge::fopen(path, "wb+"); - if (!file) { - LogPrintf("Unable to open file %s\n", path.string()); - return nullptr; - } - if (pos.nPos) { - if (fseek(file, pos.nPos, SEEK_SET)) { - LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); - fclose(file); - return nullptr; - } - } - return file; + return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE); } -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "blk", fReadOnly); +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) { + return BlockFileSeq().Open(pos, fReadOnly); } /** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "rev", fReadOnly); +static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) { + return UndoFileSeq().Open(pos, fReadOnly); } -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) +fs::path GetBlockPosFilename(const FlatFilePos &pos) { - return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile); + return BlockFileSeq().FileName(pos); } CBlockIndex * CChainState::InsertBlockIndex(const uint256& hash) @@ -3915,7 +3889,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE } for (std::set<int>::iterator it = setBlkDataFiles.begin(); it != setBlkDataFiles.end(); it++) { - CDiskBlockPos pos(*it, 0); + FlatFilePos pos(*it, 0); if (CAutoFile(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION).IsNull()) { return false; } @@ -4168,38 +4142,114 @@ bool ReplayBlocks(const CChainParams& params, CCoinsView* view) { return g_chainstate.ReplayBlocks(params, view); } -bool CChainState::RewindBlockIndex(const CChainParams& params) +//! Helper for CChainState::RewindBlockIndex +void CChainState::EraseBlockData(CBlockIndex* index) { - LOCK(cs_main); + AssertLockHeld(cs_main); + assert(!chainActive.Contains(index)); // Make sure this block isn't active + + // Reduce validity + index->nStatus = std::min<unsigned int>(index->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (index->nStatus & ~BLOCK_VALID_MASK); + // Remove have-data flags. + index->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); + // Remove storage location. + index->nFile = 0; + index->nDataPos = 0; + index->nUndoPos = 0; + // Remove various other things + index->nTx = 0; + index->nChainTx = 0; + index->nSequenceId = 0; + // Make sure it gets written. + setDirtyBlockIndex.insert(index); + // Update indexes + setBlockIndexCandidates.erase(index); + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> ret = mapBlocksUnlinked.equal_range(index->pprev); + while (ret.first != ret.second) { + if (ret.first->second == index) { + mapBlocksUnlinked.erase(ret.first++); + } else { + ++ret.first; + } + } + // Mark parent as eligible for main chain again + if (index->pprev && index->pprev->IsValid(BLOCK_VALID_TRANSACTIONS) && index->pprev->HaveTxsDownloaded()) { + setBlockIndexCandidates.insert(index->pprev); + } +} +bool CChainState::RewindBlockIndex(const CChainParams& params) +{ // Note that during -reindex-chainstate we are called with an empty chainActive! - int nHeight = 1; - while (nHeight <= chainActive.Height()) { - // Although SCRIPT_VERIFY_WITNESS is now generally enforced on all - // blocks in ConnectBlock, we don't need to go back and - // re-download/re-verify blocks from before segwit actually activated. - if (IsWitnessEnabled(chainActive[nHeight - 1], params.GetConsensus()) && !(chainActive[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { - break; + // First erase all post-segwit blocks without witness not in the main chain, + // as this can we done without costly DisconnectTip calls. Active + // blocks will be dealt with below (releasing cs_main in between). + { + LOCK(cs_main); + for (const auto& entry : mapBlockIndex) { + if (IsWitnessEnabled(entry.second->pprev, params.GetConsensus()) && !(entry.second->nStatus & BLOCK_OPT_WITNESS) && !chainActive.Contains(entry.second)) { + EraseBlockData(entry.second); + } } - nHeight++; } + // Find what height we need to reorganize to. + CBlockIndex *tip; + int nHeight = 1; + { + LOCK(cs_main); + while (nHeight <= chainActive.Height()) { + // Although SCRIPT_VERIFY_WITNESS is now generally enforced on all + // blocks in ConnectBlock, we don't need to go back and + // re-download/re-verify blocks from before segwit actually activated. + if (IsWitnessEnabled(chainActive[nHeight - 1], params.GetConsensus()) && !(chainActive[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { + break; + } + nHeight++; + } + + tip = chainActive.Tip(); + } // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 + CValidationState state; - CBlockIndex* pindex = chainActive.Tip(); - while (chainActive.Height() >= nHeight) { - if (fPruneMode && !(chainActive.Tip()->nStatus & BLOCK_HAVE_DATA)) { - // If pruning, don't try rewinding past the HAVE_DATA point; - // since older blocks can't be served anyway, there's - // no need to walk further, and trying to DisconnectTip() - // will fail (and require a needless reindex/redownload - // of the blockchain). - break; - } - if (!DisconnectTip(state, params, nullptr)) { - return error("RewindBlockIndex: unable to disconnect block at height %i (%s)", pindex->nHeight, FormatStateMessage(state)); + // Loop until the tip is below nHeight, or we reach a pruned block. + while (!ShutdownRequested()) { + { + LOCK(cs_main); + // Make sure nothing changed from under us (this won't happen because RewindBlockIndex runs before importing/network are active) + assert(tip == chainActive.Tip()); + if (tip == nullptr || tip->nHeight < nHeight) break; + if (fPruneMode && !(tip->nStatus & BLOCK_HAVE_DATA)) { + // If pruning, don't try rewinding past the HAVE_DATA point; + // since older blocks can't be served anyway, there's + // no need to walk further, and trying to DisconnectTip() + // will fail (and require a needless reindex/redownload + // of the blockchain). + break; + } + + // Disconnect block + if (!DisconnectTip(state, params, nullptr)) { + return error("RewindBlockIndex: unable to disconnect block at height %i (%s)", tip->nHeight, FormatStateMessage(state)); + } + + // Reduce validity flag and have-data flags. + // We do this after actual disconnecting, otherwise we'll end up writing the lack of data + // to disk before writing the chainstate, resulting in a failure to continue if interrupted. + // Note: If we encounter an insufficiently validated block that + // is on chainActive, it must be because we are a pruning node, and + // this block or some successor doesn't HAVE_DATA, so we were unable to + // rewind all the way. Blocks remaining on chainActive at this point + // must not have their validity reduced. + EraseBlockData(tip); + + tip = tip->pprev; } + // Make sure the queue of validation callbacks doesn't grow unboundedly. + LimitValidationInterfaceQueue(); + // Occasionally flush state to disk. if (!FlushStateToDisk(params, state, FlushStateMode::PERIODIC)) { LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", FormatStateMessage(state)); @@ -4207,53 +4257,15 @@ bool CChainState::RewindBlockIndex(const CChainParams& params) } } - // Reduce validity flag and have-data flags. - // We do this after actual disconnecting, otherwise we'll end up writing the lack of data - // to disk before writing the chainstate, resulting in a failure to continue if interrupted. - for (const auto& entry : mapBlockIndex) { - CBlockIndex* pindexIter = entry.second; - - // Note: If we encounter an insufficiently validated block that - // is on chainActive, it must be because we are a pruning node, and - // this block or some successor doesn't HAVE_DATA, so we were unable to - // rewind all the way. Blocks remaining on chainActive at this point - // must not have their validity reduced. - if (IsWitnessEnabled(pindexIter->pprev, params.GetConsensus()) && !(pindexIter->nStatus & BLOCK_OPT_WITNESS) && !chainActive.Contains(pindexIter)) { - // Reduce validity - pindexIter->nStatus = std::min<unsigned int>(pindexIter->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (pindexIter->nStatus & ~BLOCK_VALID_MASK); - // Remove have-data flags. - pindexIter->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); - // Remove storage location. - pindexIter->nFile = 0; - pindexIter->nDataPos = 0; - pindexIter->nUndoPos = 0; - // Remove various other things - pindexIter->nTx = 0; - pindexIter->nChainTx = 0; - pindexIter->nSequenceId = 0; - // Make sure it gets written. - setDirtyBlockIndex.insert(pindexIter); - // Update indexes - setBlockIndexCandidates.erase(pindexIter); - std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> ret = mapBlocksUnlinked.equal_range(pindexIter->pprev); - while (ret.first != ret.second) { - if (ret.first->second == pindexIter) { - mapBlocksUnlinked.erase(ret.first++); - } else { - ++ret.first; - } - } - } else if (pindexIter->IsValid(BLOCK_VALID_TRANSACTIONS) && pindexIter->HaveTxsDownloaded()) { - setBlockIndexCandidates.insert(pindexIter); - } - } - - if (chainActive.Tip() != nullptr) { - // We can't prune block index candidates based on our tip if we have - // no tip due to chainActive being empty! - PruneBlockIndexCandidates(); + { + LOCK(cs_main); + if (chainActive.Tip() != nullptr) { + // We can't prune block index candidates based on our tip if we have + // no tip due to chainActive being empty! + PruneBlockIndexCandidates(); - CheckBlockIndex(params.GetConsensus()); + CheckBlockIndex(params.GetConsensus()); + } } return true; @@ -4347,8 +4359,8 @@ bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) return true; try { - CBlock &block = const_cast<CBlock&>(chainparams.GenesisBlock()); - CDiskBlockPos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); + const CBlock& block = chainparams.GenesisBlock(); + FlatFilePos blockPos = SaveBlockToDisk(block, 0, chainparams, nullptr); if (blockPos.IsNull()) return error("%s: writing genesis block to disk failed", __func__); CBlockIndex *pindex = AddToBlockIndex(block); @@ -4365,10 +4377,10 @@ bool LoadGenesisBlock(const CChainParams& chainparams) return g_chainstate.LoadGenesisBlock(chainparams); } -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp) +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) - static std::multimap<uint256, CDiskBlockPos> mapBlocksUnknownParent; + static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; int64_t nStart = GetTimeMillis(); int nLoaded = 0; @@ -4454,9 +4466,9 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB while (!queue.empty()) { uint256 head = queue.front(); queue.pop_front(); - std::pair<std::multimap<uint256, CDiskBlockPos>::iterator, std::multimap<uint256, CDiskBlockPos>::iterator> range = mapBlocksUnknownParent.equal_range(head); + std::pair<std::multimap<uint256, FlatFilePos>::iterator, std::multimap<uint256, FlatFilePos>::iterator> range = mapBlocksUnknownParent.equal_range(head); while (range.first != range.second) { - std::multimap<uint256, CDiskBlockPos>::iterator it = range.first; + std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus())) { diff --git a/src/validation.h b/src/validation.h index b16d8438d7..0acca013b5 100644 --- a/src/validation.h +++ b/src/validation.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,8 +14,8 @@ #include <coins.h> #include <crypto/common.h> // for ReadLE64 #include <fs.h> -#include <protocol.h> // For CMessageHeader::MessageStartChars #include <policy/feerate.h> +#include <protocol.h> // For CMessageHeader::MessageStartChars #include <script/script_error.h> #include <sync.h> #include <versionbits.h> @@ -151,9 +151,7 @@ extern CBlockPolicyEstimator feeEstimator; extern CTxMemPool mempool; extern std::atomic_bool g_is_mempool_loaded; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; -extern BlockMap& mapBlockIndex; -extern uint64_t nLastBlockTx; -extern uint64_t nLastBlockWeight; +extern BlockMap& mapBlockIndex GUARDED_BY(cs_main); extern const std::string strMessageMagic; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; @@ -183,9 +181,6 @@ extern arith_uint256 nMinimumChainWork; /** Best header we've seen so far (used for getheaders queries' starting points). */ extern CBlockIndex *pindexBestHeader; -/** Minimum disk space required - used in CheckDiskSpace() */ -static const uint64_t nMinDiskSpace = 52428800; - /** Pruning-related variables and constants */ /** True if any block files have ever been pruned. */ extern bool fHavePruned; @@ -247,14 +242,12 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons */ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, CValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr, CBlockHeader* first_invalid = nullptr) LOCKS_EXCLUDED(cs_main); -/** Check whether enough disk space is available for an incoming block */ -bool CheckDiskSpace(uint64_t nAdditionalBytes = 0, bool blocks_dir = false); /** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly = false); +FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); /** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix); +fs::path GetBlockPosFilename(const FlatFilePos &pos); /** Import blocks from an external file */ -bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp = nullptr); +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos *dbp = nullptr); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(const CChainParams& chainparams); /** Load the block tree and coins database from disk, @@ -288,7 +281,7 @@ uint64_t CalculateCurrentUsage(); /** * Mark one block file as pruned. */ -void PruneOneBlockFile(const int fileNumber); +void PruneOneBlockFile(const int fileNumber) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** * Actually unlink the specified files @@ -393,9 +386,9 @@ void InitScriptExecutionCache(); /** Functions for disk access for blocks */ -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams); +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& message_start); +bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start); bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); /** Functions for validating blocks and updating the block tree */ @@ -450,7 +443,7 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex) LOCKS_EXCLUDED(cs_main); /** Mark a block as invalid. */ -bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex); /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 1dc78255f6..a255177e36 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -182,7 +182,7 @@ bool CCryptoKeyStore::Unlock(const CKeyingMaterial& vMasterKeyIn, bool accept_no if (!SetCrypted()) return false; - bool keyPass = false; + bool keyPass = mapCryptedKeys.empty(); // Always pass when there are no encrypted keys bool keyFail = false; CryptedKeyMap::const_iterator mi = mapCryptedKeys.begin(); for (; mi != mapCryptedKeys.end(); ++mi) diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index c64a85c910..99d880daa0 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -397,7 +397,7 @@ bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, std::string& er fs::path walletDir = env->Directory(); LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(nullptr, nullptr, nullptr)); - LogPrintf("Using wallet %s\n", walletFile); + LogPrintf("Using wallet %s\n", file_path.string()); // Wallet file must be a plain filename without a directory if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 7a71aea715..a1c3a21d4b 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -27,9 +27,7 @@ static feebumper::Result PreconditionChecks(interfaces::Chain::Lock& locked_chai } { - LOCK(mempool.cs); - auto it_mp = mempool.mapTx.find(wtx.GetHash()); - if (it_mp != mempool.mapTx.end() && it_mp->GetCountWithDescendants() > 1) { + if (wallet->chain().hasDescendantsInMempool(wtx.GetHash())) { errors.push_back("Transaction has descendants in the mempool"); return feebumper::Result::INVALID_PARAMETER; } @@ -146,7 +144,7 @@ Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoin new_fee = total_fee; nNewFeeRate = CFeeRate(total_fee, maxNewTxSize); } else { - new_fee = GetMinimumFee(*wallet, maxNewTxSize, coin_control, mempool, ::feeEstimator, nullptr /* FeeCalculation */); + new_fee = GetMinimumFee(*wallet, maxNewTxSize, coin_control, nullptr /* FeeCalculation */); nNewFeeRate = CFeeRate(new_fee, maxNewTxSize); // New fee rate must be at least old rate + minimum incremental relay rate @@ -197,7 +195,7 @@ Result CreateTransaction(const CWallet* wallet, const uint256& txid, const CCoin // If the output would become dust, discard it (converting the dust to fee) poutput->nValue -= nDelta; - if (poutput->nValue <= GetDustThreshold(*poutput, GetDiscardRate(*wallet, ::feeEstimator))) { + if (poutput->nValue <= GetDustThreshold(*poutput, GetDiscardRate(*wallet))) { wallet->WalletLogPrintf("Bumping fee and discarding dust output\n"); new_fee += poutput->nValue; mtx.vout.erase(mtx.vout.begin() + nOutput); @@ -247,7 +245,7 @@ Result CommitTransaction(CWallet* wallet, const uint256& txid, CMutableTransacti CReserveKey reservekey(wallet); CValidationState state; - if (!wallet->CommitTransaction(tx, std::move(mapValue), oldWtx.vOrderForm, reservekey, g_connman.get(), state)) { + if (!wallet->CommitTransaction(tx, std::move(mapValue), oldWtx.vOrderForm, reservekey, state)) { // NOTE: CommitTransaction never returns false, so this should never happen. errors.push_back(strprintf("The transaction was rejected: %s", FormatStateMessage(state))); return Result::WALLET_ERROR; diff --git a/src/wallet/fees.cpp b/src/wallet/fees.cpp index 9e2984ff05..545adaebc1 100644 --- a/src/wallet/fees.cpp +++ b/src/wallet/fees.cpp @@ -6,7 +6,6 @@ #include <wallet/fees.h> #include <policy/policy.h> -#include <txmempool.h> #include <util/system.h> #include <validation.h> #include <wallet/coincontrol.h> @@ -19,9 +18,9 @@ CAmount GetRequiredFee(const CWallet& wallet, unsigned int nTxBytes) } -CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc) +CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, FeeCalculation* feeCalc) { - CAmount fee_needed = GetMinimumFeeRate(wallet, coin_control, pool, estimator, feeCalc).GetFee(nTxBytes); + CAmount fee_needed = GetMinimumFeeRate(wallet, coin_control, feeCalc).GetFee(nTxBytes); // Always obey the maximum if (fee_needed > maxTxFee) { fee_needed = maxTxFee; @@ -35,7 +34,7 @@ CFeeRate GetRequiredFeeRate(const CWallet& wallet) return std::max(wallet.m_min_fee, ::minRelayTxFee); } -CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc) +CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, FeeCalculation* feeCalc) { /* User control of how to calculate fee uses the following parameter precedence: 1. coin_control.m_feerate @@ -64,7 +63,7 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr if (coin_control.m_fee_mode == FeeEstimateMode::CONSERVATIVE) conservative_estimate = true; else if (coin_control.m_fee_mode == FeeEstimateMode::ECONOMICAL) conservative_estimate = false; - feerate_needed = estimator.estimateSmartFee(target, feeCalc, conservative_estimate); + feerate_needed = wallet.chain().estimateSmartFee(target, conservative_estimate, feeCalc); if (feerate_needed == CFeeRate(0)) { // if we don't have enough data for estimateSmartFee, then use fallback fee feerate_needed = wallet.m_fallback_fee; @@ -74,7 +73,7 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr if (wallet.m_fallback_fee == CFeeRate(0)) return feerate_needed; } // Obey mempool min fee when using smart fee estimation - CFeeRate min_mempool_feerate = pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + CFeeRate min_mempool_feerate = wallet.chain().mempoolMinFee(); if (feerate_needed < min_mempool_feerate) { feerate_needed = min_mempool_feerate; if (feeCalc) feeCalc->reason = FeeReason::MEMPOOL_MIN; @@ -90,10 +89,10 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr return feerate_needed; } -CFeeRate GetDiscardRate(const CWallet& wallet, const CBlockPolicyEstimator& estimator) +CFeeRate GetDiscardRate(const CWallet& wallet) { - unsigned int highest_target = estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); - CFeeRate discard_rate = estimator.estimateSmartFee(highest_target, nullptr /* FeeCalculation */, false /* conservative */); + unsigned int highest_target = wallet.chain().estimateMaxBlocks(); + CFeeRate discard_rate = wallet.chain().estimateSmartFee(highest_target, false /* conservative */); // Don't let discard_rate be greater than longest possible fee estimate if we get a valid fee estimate discard_rate = (discard_rate == CFeeRate(0)) ? wallet.m_discard_rate : std::min(discard_rate, wallet.m_discard_rate); // Discard rate must be at least dustRelayFee diff --git a/src/wallet/fees.h b/src/wallet/fees.h index 6bfee456c0..434f211dc2 100644 --- a/src/wallet/fees.h +++ b/src/wallet/fees.h @@ -8,10 +8,8 @@ #include <amount.h> -class CBlockPolicyEstimator; class CCoinControl; class CFeeRate; -class CTxMemPool; class CWallet; struct FeeCalculation; @@ -25,7 +23,7 @@ CAmount GetRequiredFee(const CWallet& wallet, unsigned int nTxBytes); * Estimate the minimum fee considering user set parameters * and the required fee */ -CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc); +CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinControl& coin_control, FeeCalculation* feeCalc); /** * Return the minimum required feerate taking into account the @@ -37,11 +35,11 @@ CFeeRate GetRequiredFeeRate(const CWallet& wallet); * Estimate the minimum fee rate considering user set parameters * and the required fee */ -CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, FeeCalculation* feeCalc); +CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_control, FeeCalculation* feeCalc); /** * Return the maximum feerate for discarding change. */ -CFeeRate GetDiscardRate(const CWallet& wallet, const CBlockPolicyEstimator& estimator); +CFeeRate GetDiscardRate(const CWallet& wallet); #endif // BITCOIN_WALLET_FEES_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 87cd264c3d..7ad343c15f 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -127,21 +127,6 @@ bool WalletInit::ParameterInteraction() const InitWarning(AmountHighWarn("-minrelaytxfee") + " " + _("The wallet will avoid paying less than the minimum relay fee.")); - if (gArgs.IsArgSet("-maxtxfee")) - { - CAmount nMaxFee = 0; - if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) - return InitError(AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", ""))); - if (nMaxFee > HIGH_MAX_TX_FEE) - InitWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); - maxTxFee = nMaxFee; - if (CFeeRate(maxTxFee, 1000) < ::minRelayTxFee) - { - return InitError(strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), - gArgs.GetArg("-maxtxfee", ""), ::minRelayTxFee.ToString())); - } - } - return true; } @@ -153,12 +138,15 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal // The canonical path cleans the path, preventing >1 Berkeley environment instances for the same directory fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error); if (error || !fs::exists(wallet_dir)) { - return InitError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); + return false; } else if (!fs::is_directory(wallet_dir)) { - return InitError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); + return false; // The canonical path transforms relative paths into absolute ones, so we check the non-canonical version } else if (!wallet_dir.is_absolute()) { - return InitError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); + return false; } gArgs.ForceSetArg("-walletdir", canonical_wallet_dir.string()); } @@ -179,14 +167,15 @@ bool VerifyWallets(interfaces::Chain& chain, const std::vector<std::string>& wal WalletLocation location(wallet_file); if (!wallet_paths.insert(location.GetPath()).second) { - return InitError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified."), wallet_file)); + chain.initError(strprintf(_("Error loading wallet %s. Duplicate -wallet filename specified."), wallet_file)); + return false; } std::string error_string; std::string warning_string; bool verify_success = CWallet::Verify(chain, location, salvage_wallet, error_string, warning_string); - if (!error_string.empty()) InitError(error_string); - if (!warning_string.empty()) InitWarning(warning_string); + if (!error_string.empty()) chain.initError(error_string); + if (!warning_string.empty()) chain.initWarning(warning_string); if (!verify_success) return false; } diff --git a/src/wallet/psbtwallet.cpp b/src/wallet/psbtwallet.cpp new file mode 100644 index 0000000000..1b17b09763 --- /dev/null +++ b/src/wallet/psbtwallet.cpp @@ -0,0 +1,60 @@ +// Copyright (c) 2009-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <wallet/psbtwallet.h> + +TransactionError FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& psbtx, bool& complete, int sighash_type, bool sign, bool bip32derivs) +{ + LOCK(pwallet->cs_wallet); + // Get all of the previous transactions + complete = true; + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + const CTxIn& txin = psbtx.tx->vin[i]; + PSBTInput& input = psbtx.inputs.at(i); + + if (PSBTInputSigned(input)) { + continue; + } + + // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. + if (!input.IsSane()) { + return TransactionError::INVALID_PSBT; + } + + // If we have no utxo, grab it from the wallet. + if (!input.non_witness_utxo && input.witness_utxo.IsNull()) { + const uint256& txhash = txin.prevout.hash; + const auto it = pwallet->mapWallet.find(txhash); + if (it != pwallet->mapWallet.end()) { + const CWalletTx& wtx = it->second; + // We only need the non_witness_utxo, which is a superset of the witness_utxo. + // The signing code will switch to the smaller witness_utxo if this is ok. + input.non_witness_utxo = wtx.tx; + } + } + + // Get the Sighash type + if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { + return TransactionError::SIGHASH_MISMATCH; + } + + complete &= SignPSBTInput(HidingSigningProvider(pwallet, !sign, !bip32derivs), psbtx, i, sighash_type); + } + + // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change + for (unsigned int i = 0; i < psbtx.tx->vout.size(); ++i) { + const CTxOut& out = psbtx.tx->vout.at(i); + PSBTOutput& psbt_out = psbtx.outputs.at(i); + + // Fill a SignatureData with output info + SignatureData sigdata; + psbt_out.FillSignatureData(sigdata); + + MutableTransactionSignatureCreator creator(psbtx.tx.get_ptr(), 0, out.nValue, 1); + ProduceSignature(HidingSigningProvider(pwallet, true, !bip32derivs), creator, out.scriptPubKey, sigdata); + psbt_out.FromSignatureData(sigdata); + } + + return TransactionError::OK; +} diff --git a/src/wallet/psbtwallet.h b/src/wallet/psbtwallet.h new file mode 100644 index 0000000000..a24a0967d2 --- /dev/null +++ b/src/wallet/psbtwallet.h @@ -0,0 +1,34 @@ +// Copyright (c) 2009-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_PSBTWALLET_H +#define BITCOIN_WALLET_PSBTWALLET_H + +#include <node/transaction.h> +#include <psbt.h> +#include <primitives/transaction.h> +#include <wallet/wallet.h> + +/** + * Fills out a PSBT with information from the wallet. Fills in UTXOs if we have + * them. Tries to sign if sign=true. Sets `complete` if the PSBT is now complete + * (i.e. has all required signatures or signature-parts, and is ready to + * finalize.) Sets `error` and returns false if something goes wrong. + * + * @param[in] pwallet pointer to a wallet + * @param[in] &psbtx reference to PartiallySignedTransaction to fill in + * @param[out] &complete indicates whether the PSBT is now complete + * @param[in] sighash_type the sighash type to use when signing (if PSBT does not specify) + * @param[in] sign whether to sign or not + * @param[in] bip32derivs whether to fill in bip32 derivation information if available + * return error + */ +NODISCARD TransactionError FillPSBT(const CWallet* pwallet, + PartiallySignedTransaction& psbtx, + bool& complete, + int sighash_type = 1 /* SIGHASH_ALL */, + bool sign = true, + bool bip32derivs = false); + +#endif // BITCOIN_WALLET_PSBTWALLET_H diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 32c36ceaeb..684d900478 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -9,9 +9,11 @@ #include <merkleblock.h> #include <rpc/server.h> #include <rpc/util.h> +#include <script/descriptor.h> #include <script/script.h> #include <script/standard.h> #include <sync.h> +#include <util/bip32.h> #include <util/system.h> #include <util/time.h> #include <validation.h> @@ -66,7 +68,7 @@ static std::string DecodeDumpString(const std::string &str) { return ret.str(); } -static bool GetWalletAddressesForKey(CWallet * const pwallet, const CKeyID &keyid, std::string &strAddr, std::string &strLabel) +static bool GetWalletAddressesForKey(CWallet* const pwallet, const CKeyID& keyid, std::string& strAddr, std::string& strLabel) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { bool fLabelFound = false; CKey key; @@ -115,9 +117,9 @@ UniValue importprivkey(const JSONRPCRequest& request) "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported key exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n", { - {"privkey", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The private key (see dumpprivkey)"}, - {"label", RPCArg::Type::STR, /* opt */ true, /* default_val */ "current label if address exists, otherwise \"\"", "An optional label"}, - {"rescan", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "Rescan the wallet for transactions"}, + {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key (see dumpprivkey)"}, + {"label", RPCArg::Type::STR, /* default */ "current label if address exists, otherwise \"\"", "An optional label"}, + {"rescan", RPCArg::Type::BOOL, /* default */ "true", "Rescan the wallet for transactions"}, }, RPCResults{}, RPCExamples{ @@ -155,8 +157,9 @@ UniValue importprivkey(const JSONRPCRequest& request) if (!request.params[2].isNull()) fRescan = request.params[2].get_bool(); - if (fRescan && fPruneMode) + if (fRescan && pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + } if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); @@ -285,10 +288,10 @@ UniValue importaddress(const JSONRPCRequest& request) "\nNote: If you import a non-standard raw script in hex form, outputs sending to it will be treated\n" "as change, and not show up in many RPCs.\n", { - {"address", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The Bitcoin address (or hex-encoded script)"}, - {"label", RPCArg::Type::STR, /* opt */ true, /* default_val */ "\"\"", "An optional label"}, - {"rescan", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "Rescan the wallet for transactions"}, - {"p2sh", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Add the P2SH version of the script as well"}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The Bitcoin address (or hex-encoded script)"}, + {"label", RPCArg::Type::STR, /* default */ "\"\"", "An optional label"}, + {"rescan", RPCArg::Type::BOOL, /* default */ "true", "Rescan the wallet for transactions"}, + {"p2sh", RPCArg::Type::BOOL, /* default */ "false", "Add the P2SH version of the script as well"}, }, RPCResults{}, RPCExamples{ @@ -311,8 +314,9 @@ UniValue importaddress(const JSONRPCRequest& request) if (!request.params[2].isNull()) fRescan = request.params[2].get_bool(); - if (fRescan && fPruneMode) + if (fRescan && pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + } WalletRescanReserver reserver(pwallet); if (fRescan && !reserver.reserve()) { @@ -363,8 +367,8 @@ UniValue importprunedfunds(const JSONRPCRequest& request) RPCHelpMan{"importprunedfunds", "\nImports funds without rescan. Corresponding address or script must previously be included in wallet. Aimed towards pruned wallets. The end-user is responsible to import additional transactions that subsequently spend the imported outputs or rescan after the point in the blockchain the transaction is included.\n", { - {"rawtransaction", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "A raw transaction in hex funding an already-existing address in wallet"}, - {"txoutproof", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The hex output from gettxoutproof that contains the transaction"}, + {"rawtransaction", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "A raw transaction in hex funding an already-existing address in wallet"}, + {"txoutproof", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex output from gettxoutproof that contains the transaction"}, }, RPCResults{}, RPCExamples{""}, @@ -430,7 +434,7 @@ UniValue removeprunedfunds(const JSONRPCRequest& request) RPCHelpMan{"removeprunedfunds", "\nDeletes the specified transaction from the wallet. Meant for use with pruned wallets and as a companion to importprunedfunds. This will affect wallet balances.\n", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The hex-encoded id of the transaction you are deleting"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex-encoded id of the transaction you are deleting"}, }, RPCResults{}, RPCExamples{ @@ -474,9 +478,9 @@ UniValue importpubkey(const JSONRPCRequest& request) "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported pubkey exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n", { - {"pubkey", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The hex-encoded public key"}, - {"label", RPCArg::Type::STR, /* opt */ true, /* default_val */ "\"\"", "An optional label"}, - {"rescan", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "Rescan the wallet for transactions"}, + {"pubkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The hex-encoded public key"}, + {"label", RPCArg::Type::STR, /* default */ "\"\"", "An optional label"}, + {"rescan", RPCArg::Type::BOOL, /* default */ "true", "Rescan the wallet for transactions"}, }, RPCResults{}, RPCExamples{ @@ -499,8 +503,9 @@ UniValue importpubkey(const JSONRPCRequest& request) if (!request.params[2].isNull()) fRescan = request.params[2].get_bool(); - if (fRescan && fPruneMode) + if (fRescan && pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled in pruned mode"); + } WalletRescanReserver reserver(pwallet); if (fRescan && !reserver.reserve()) { @@ -547,7 +552,7 @@ UniValue importwallet(const JSONRPCRequest& request) RPCHelpMan{"importwallet", "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n", { - {"filename", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The wallet file"}, + {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet file"}, }, RPCResults{}, RPCExamples{ @@ -560,8 +565,9 @@ UniValue importwallet(const JSONRPCRequest& request) }, }.ToString()); - if (fPruneMode) + if (pwallet->chain().getPruneMode()) { throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled in pruned mode"); + } WalletRescanReserver reserver(pwallet); if (!reserver.reserve()) { @@ -708,7 +714,7 @@ UniValue dumpprivkey(const JSONRPCRequest& request) "\nReveals the private key corresponding to 'address'.\n" "Then the importprivkey can be used with this output\n", { - {"address", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The bitcoin address for the private key"}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address for the private key"}, }, RPCResult{ "\"key\" (string) The private key\n" @@ -758,7 +764,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) "Note that if your wallet contains keys which are not derived from your HD seed (e.g. imported keys), these are not covered by\n" "only backing up the seed itself, and must be backed up too (e.g. ensure you back up the whole dumpfile).\n", { - {"filename", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The filename with path (either absolute or relative to bitcoind)"}, + {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The filename with path (either absolute or relative to bitcoind)"}, }, RPCResult{ "{ (json object)\n" @@ -849,7 +855,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) } else { file << "change=1"; } - file << strprintf(" # addr=%s%s\n", strAddr, (pwallet->mapKeyMetadata[keyid].hdKeypath.size() > 0 ? " hdkeypath="+pwallet->mapKeyMetadata[keyid].hdKeypath : "")); + file << strprintf(" # addr=%s%s\n", strAddr, (pwallet->mapKeyMetadata[keyid].has_key_origin ? " hdkeypath="+WriteHDKeypath(pwallet->mapKeyMetadata[keyid].key_origin.path) : "")); } } file << "\n"; @@ -886,6 +892,7 @@ struct ImportData // Output data std::set<CScript> import_scripts; std::map<CKeyID, bool> used_keys; //!< Import these private keys if available (the value indicates whether if the key is required for solvability) + std::map<CKeyID, KeyOriginInfo> key_origins; }; enum class ScriptContext @@ -964,159 +971,281 @@ static std::string RecurseImportData(const CScript& script, ImportData& import_d } } -static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +static UniValue ProcessImportLegacy(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data, std::vector<CKeyID>& ordered_pubkeys) { UniValue warnings(UniValue::VARR); - UniValue result(UniValue::VOBJ); - try { - // First ensure scriptPubKey has either a script or JSON with "address" string - const UniValue& scriptPubKey = data["scriptPubKey"]; - bool isScript = scriptPubKey.getType() == UniValue::VSTR; - if (!isScript && !(scriptPubKey.getType() == UniValue::VOBJ && scriptPubKey.exists("address"))) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "scriptPubKey must be string with script or JSON with address string"); + // First ensure scriptPubKey has either a script or JSON with "address" string + const UniValue& scriptPubKey = data["scriptPubKey"]; + bool isScript = scriptPubKey.getType() == UniValue::VSTR; + if (!isScript && !(scriptPubKey.getType() == UniValue::VOBJ && scriptPubKey.exists("address"))) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "scriptPubKey must be string with script or JSON with address string"); + } + const std::string& output = isScript ? scriptPubKey.get_str() : scriptPubKey["address"].get_str(); + + // Optional fields. + const std::string& strRedeemScript = data.exists("redeemscript") ? data["redeemscript"].get_str() : ""; + const std::string& witness_script_hex = data.exists("witnessscript") ? data["witnessscript"].get_str() : ""; + const UniValue& pubKeys = data.exists("pubkeys") ? data["pubkeys"].get_array() : UniValue(); + const UniValue& keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); + const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; + const bool watchOnly = data.exists("watchonly") ? data["watchonly"].get_bool() : false; + + if (data.exists("range")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for a non-descriptor import"); + } + + // Generate the script and destination for the scriptPubKey provided + CScript script; + if (!isScript) { + CTxDestination dest = DecodeDestination(output); + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address \"" + output + "\""); + } + script = GetScriptForDestination(dest); + } else { + if (!IsHex(output)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid scriptPubKey \"" + output + "\""); } - const std::string& output = isScript ? scriptPubKey.get_str() : scriptPubKey["address"].get_str(); + std::vector<unsigned char> vData(ParseHex(output)); + script = CScript(vData.begin(), vData.end()); + CTxDestination dest; + if (!ExtractDestination(script, dest) && !internal) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal must be set to true for nonstandard scriptPubKey imports."); + } + } + script_pub_keys.emplace(script); - // Optional fields. - const std::string& strRedeemScript = data.exists("redeemscript") ? data["redeemscript"].get_str() : ""; - const std::string& witness_script_hex = data.exists("witnessscript") ? data["witnessscript"].get_str() : ""; - const UniValue& pubKeys = data.exists("pubkeys") ? data["pubkeys"].get_array() : UniValue(); - const UniValue& keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); - const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; - const bool watchOnly = data.exists("watchonly") ? data["watchonly"].get_bool() : false; - const std::string& label = data.exists("label") ? data["label"].get_str() : ""; + // Parse all arguments + if (strRedeemScript.size()) { + if (!IsHex(strRedeemScript)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid redeem script \"" + strRedeemScript + "\": must be hex string"); + } + auto parsed_redeemscript = ParseHex(strRedeemScript); + import_data.redeemscript = MakeUnique<CScript>(parsed_redeemscript.begin(), parsed_redeemscript.end()); + } + if (witness_script_hex.size()) { + if (!IsHex(witness_script_hex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid witness script \"" + witness_script_hex + "\": must be hex string"); + } + auto parsed_witnessscript = ParseHex(witness_script_hex); + import_data.witnessscript = MakeUnique<CScript>(parsed_witnessscript.begin(), parsed_witnessscript.end()); + } + for (size_t i = 0; i < pubKeys.size(); ++i) { + const auto& str = pubKeys[i].get_str(); + if (!IsHex(str)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" must be a hex string"); + } + auto parsed_pubkey = ParseHex(str); + CPubKey pubkey(parsed_pubkey.begin(), parsed_pubkey.end()); + if (!pubkey.IsFullyValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" is not a valid public key"); + } + pubkey_map.emplace(pubkey.GetID(), pubkey); + ordered_pubkeys.push_back(pubkey.GetID()); + } + for (size_t i = 0; i < keys.size(); ++i) { + const auto& str = keys[i].get_str(); + CKey key = DecodeSecret(str); + if (!key.IsValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); + } + CPubKey pubkey = key.GetPubKey(); + CKeyID id = pubkey.GetID(); + if (pubkey_map.count(id)) { + pubkey_map.erase(id); + } + privkey_map.emplace(id, key); + } - // If private keys are disabled, abort if private keys are being imported - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !keys.isNull()) { - throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); + + // Verify and process input data + have_solving_data = import_data.redeemscript || import_data.witnessscript || pubkey_map.size() || privkey_map.size(); + if (have_solving_data) { + // Match up data in import_data with the scriptPubKey in script. + auto error = RecurseImportData(script, import_data, ScriptContext::TOP); + + // Verify whether the watchonly option corresponds to the availability of private keys. + bool spendable = std::all_of(import_data.used_keys.begin(), import_data.used_keys.end(), [&](const std::pair<CKeyID, bool>& used_key){ return privkey_map.count(used_key.first) > 0; }); + if (!watchOnly && !spendable) { + warnings.push_back("Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."); + } + if (watchOnly && spendable) { + warnings.push_back("All private keys are provided, outputs will be considered spendable. If this is intentional, do not specify the watchonly flag."); } - // Generate the script and destination for the scriptPubKey provided - CScript script; - CTxDestination dest; - if (!isScript) { - dest = DecodeDestination(output); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address \"" + output + "\""); + // Check that all required keys for solvability are provided. + if (error.empty()) { + for (const auto& require_key : import_data.used_keys) { + if (!require_key.second) continue; // Not a required key + if (pubkey_map.count(require_key.first) == 0 && privkey_map.count(require_key.first) == 0) { + error = "some required keys are missing"; + } } - script = GetScriptForDestination(dest); + } + + if (!error.empty()) { + warnings.push_back("Importing as non-solvable: " + error + ". If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript."); + import_data = ImportData(); + pubkey_map.clear(); + privkey_map.clear(); + have_solving_data = false; } else { - if (!IsHex(output)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid scriptPubKey \"" + output + "\""); + // RecurseImportData() removes any relevant redeemscript/witnessscript from import_data, so we can use that to discover if a superfluous one was provided. + if (import_data.redeemscript) warnings.push_back("Ignoring redeemscript as this is not a P2SH script."); + if (import_data.witnessscript) warnings.push_back("Ignoring witnessscript as this is not a (P2SH-)P2WSH script."); + for (auto it = privkey_map.begin(); it != privkey_map.end(); ) { + auto oldit = it++; + if (import_data.used_keys.count(oldit->first) == 0) { + warnings.push_back("Ignoring irrelevant private key."); + privkey_map.erase(oldit); + } } - std::vector<unsigned char> vData(ParseHex(output)); - script = CScript(vData.begin(), vData.end()); - if (!ExtractDestination(script, dest) && !internal) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal must be set to true for nonstandard scriptPubKey imports."); + for (auto it = pubkey_map.begin(); it != pubkey_map.end(); ) { + auto oldit = it++; + auto key_data_it = import_data.used_keys.find(oldit->first); + if (key_data_it == import_data.used_keys.end() || !key_data_it->second) { + warnings.push_back("Ignoring public key \"" + HexStr(oldit->first) + "\" as it doesn't appear inside P2PKH or P2WPKH."); + pubkey_map.erase(oldit); + } } } + } - // Parse all arguments - ImportData import_data; - if (strRedeemScript.size()) { - if (!IsHex(strRedeemScript)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid redeem script \"" + strRedeemScript + "\": must be hex string"); - } - auto parsed_redeemscript = ParseHex(strRedeemScript); - import_data.redeemscript = MakeUnique<CScript>(parsed_redeemscript.begin(), parsed_redeemscript.end()); + return warnings; +} + +static UniValue ProcessImportDescriptor(ImportData& import_data, std::map<CKeyID, CPubKey>& pubkey_map, std::map<CKeyID, CKey>& privkey_map, std::set<CScript>& script_pub_keys, bool& have_solving_data, const UniValue& data, std::vector<CKeyID>& ordered_pubkeys) +{ + UniValue warnings(UniValue::VARR); + + const std::string& descriptor = data["desc"].get_str(); + FlatSigningProvider keys; + auto parsed_desc = Parse(descriptor, keys, /* require_checksum = */ true); + if (!parsed_desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Descriptor is invalid"); + } + + have_solving_data = parsed_desc->IsSolvable(); + const bool watch_only = data.exists("watchonly") ? data["watchonly"].get_bool() : false; + + int64_t range_start = 0, range_end = 0; + if (!parsed_desc->IsRange() && data.exists("range")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for an un-ranged descriptor"); + } else if (parsed_desc->IsRange()) { + if (!data.exists("range")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Descriptor is ranged, please specify the range"); } - if (witness_script_hex.size()) { - if (!IsHex(witness_script_hex)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid witness script \"" + witness_script_hex + "\": must be hex string"); - } - auto parsed_witnessscript = ParseHex(witness_script_hex); - import_data.witnessscript = MakeUnique<CScript>(parsed_witnessscript.begin(), parsed_witnessscript.end()); + auto range = ParseRange(data["range"]); + range_start = range.first; + range_end = range.second; + if (range_start < 0 || (range_end >> 31) != 0 || range_end - range_start >= 1000000) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid descriptor range specified"); } - std::map<CKeyID, CPubKey> pubkey_map; - for (size_t i = 0; i < pubKeys.size(); ++i) { - const auto& str = pubKeys[i].get_str(); - if (!IsHex(str)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" must be a hex string"); - } - auto parsed_pubkey = ParseHex(str); - CPubKey pubkey(parsed_pubkey.begin(), parsed_pubkey.end()); - if (!pubkey.IsFullyValid()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Pubkey \"" + str + "\" is not a valid public key"); - } - pubkey_map.emplace(pubkey.GetID(), pubkey); + } + + const UniValue& priv_keys = data.exists("keys") ? data["keys"].get_array() : UniValue(); + + // Expand all descriptors to get public keys and scripts. + // TODO: get private keys from descriptors too + for (int i = range_start; i <= range_end; ++i) { + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + parsed_desc->Expand(i, keys, scripts_temp, out_keys); + std::copy(scripts_temp.begin(), scripts_temp.end(), std::inserter(script_pub_keys, script_pub_keys.end())); + for (const auto& key_pair : out_keys.pubkeys) { + ordered_pubkeys.push_back(key_pair.first); } - std::map<CKeyID, CKey> privkey_map; - for (size_t i = 0; i < keys.size(); ++i) { - const auto& str = keys[i].get_str(); - CKey key = DecodeSecret(str); - if (!key.IsValid()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); - } - CPubKey pubkey = key.GetPubKey(); - CKeyID id = pubkey.GetID(); - if (pubkey_map.count(id)) { - pubkey_map.erase(id); - } + + for (const auto& x : out_keys.scripts) { + import_data.import_scripts.emplace(x.second); + } + + std::copy(out_keys.pubkeys.begin(), out_keys.pubkeys.end(), std::inserter(pubkey_map, pubkey_map.end())); + import_data.key_origins.insert(out_keys.origins.begin(), out_keys.origins.end()); + } + + for (size_t i = 0; i < priv_keys.size(); ++i) { + const auto& str = priv_keys[i].get_str(); + CKey key = DecodeSecret(str); + if (!key.IsValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid private key encoding"); + } + CPubKey pubkey = key.GetPubKey(); + CKeyID id = pubkey.GetID(); + + // Check if this private key corresponds to a public key from the descriptor + if (!pubkey_map.count(id)) { + warnings.push_back("Ignoring irrelevant private key."); + } else { privkey_map.emplace(id, key); } + } + + // Check if all the public keys have corresponding private keys in the import for spendability. + // This does not take into account threshold multisigs which could be spendable without all keys. + // Thus, threshold multisigs without all keys will be considered not spendable here, even if they are, + // perhaps triggering a false warning message. This is consistent with the current wallet IsMine check. + bool spendable = std::all_of(pubkey_map.begin(), pubkey_map.end(), + [&](const std::pair<CKeyID, CPubKey>& used_key) { + return privkey_map.count(used_key.first) > 0; + }); + if (!watch_only && !spendable) { + warnings.push_back("Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."); + } + if (watch_only && spendable) { + warnings.push_back("All private keys are provided, outputs will be considered spendable. If this is intentional, do not specify the watchonly flag."); + } + + return warnings; +} + +static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + UniValue warnings(UniValue::VARR); + UniValue result(UniValue::VOBJ); + try { + const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; // Internal addresses should not have a label if (internal && data.exists("label")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal addresses should not have a label"); } + const std::string& label = data.exists("label") ? data["label"].get_str() : ""; + const bool add_keypool = data.exists("keypool") ? data["keypool"].get_bool() : false; - // Verify and process input data - bool have_solving_data = import_data.redeemscript || import_data.witnessscript || pubkey_map.size() || privkey_map.size(); - if (have_solving_data) { - // Match up data in import_data with the scriptPubKey in script. - auto error = RecurseImportData(script, import_data, ScriptContext::TOP); - - // Verify whether the watchonly option corresponds to the availability of private keys. - bool spendable = std::all_of(import_data.used_keys.begin(), import_data.used_keys.end(), [&](const std::pair<CKeyID, bool>& used_key){ return privkey_map.count(used_key.first) > 0; }); - if (!watchOnly && !spendable) { - warnings.push_back("Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."); - } - if (watchOnly && spendable) { - warnings.push_back("All private keys are provided, outputs will be considered spendable. If this is intentional, do not specify the watchonly flag."); - } + // Add to keypool only works with privkeys disabled + if (add_keypool && !pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Keys can only be imported to the keypool when private keys are disabled"); + } - // Check that all required keys for solvability are provided. - if (error.empty()) { - for (const auto& require_key : import_data.used_keys) { - if (!require_key.second) continue; // Not a required key - if (pubkey_map.count(require_key.first) == 0 && privkey_map.count(require_key.first) == 0) { - error = "some required keys are missing"; - } - } - } + ImportData import_data; + std::map<CKeyID, CPubKey> pubkey_map; + std::map<CKeyID, CKey> privkey_map; + std::set<CScript> script_pub_keys; + std::vector<CKeyID> ordered_pubkeys; + bool have_solving_data; + + if (data.exists("scriptPubKey") && data.exists("desc")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Both a descriptor and a scriptPubKey should not be provided."); + } else if (data.exists("scriptPubKey")) { + warnings = ProcessImportLegacy(import_data, pubkey_map, privkey_map, script_pub_keys, have_solving_data, data, ordered_pubkeys); + } else if (data.exists("desc")) { + warnings = ProcessImportDescriptor(import_data, pubkey_map, privkey_map, script_pub_keys, have_solving_data, data, ordered_pubkeys); + } else { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Either a descriptor or scriptPubKey must be provided."); + } - if (!error.empty()) { - warnings.push_back("Importing as non-solvable: " + error + ". If this is intentional, don't provide any keys, pubkeys, witnessscript, or redeemscript."); - import_data = ImportData(); - pubkey_map.clear(); - privkey_map.clear(); - have_solving_data = false; - } else { - // RecurseImportData() removes any relevant redeemscript/witnessscript from import_data, so we can use that to discover if a superfluous one was provided. - if (import_data.redeemscript) warnings.push_back("Ignoring redeemscript as this is not a P2SH script."); - if (import_data.witnessscript) warnings.push_back("Ignoring witnessscript as this is not a (P2SH-)P2WSH script."); - for (auto it = privkey_map.begin(); it != privkey_map.end(); ) { - auto oldit = it++; - if (import_data.used_keys.count(oldit->first) == 0) { - warnings.push_back("Ignoring irrelevant private key."); - privkey_map.erase(oldit); - } - } - for (auto it = pubkey_map.begin(); it != pubkey_map.end(); ) { - auto oldit = it++; - auto key_data_it = import_data.used_keys.find(oldit->first); - if (key_data_it == import_data.used_keys.end() || !key_data_it->second) { - warnings.push_back("Ignoring public key \"" + HexStr(oldit->first) + "\" as it doesn't appear inside P2PKH or P2WPKH."); - pubkey_map.erase(oldit); - } - } - } + // If private keys are disabled, abort if private keys are being imported + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !privkey_map.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); } // Check whether we have any work to do - if (::IsMine(*pwallet, script) & ISMINE_SPENDABLE) { - throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); + for (const CScript& script : script_pub_keys) { + if (::IsMine(*pwallet, script) & ISMINE_SPENDABLE) { + throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script (\"" + HexStr(script.begin(), script.end()) + "\")"); + } } // All good, time to import @@ -1124,37 +1253,54 @@ static UniValue ProcessImport(CWallet * const pwallet, const UniValue& data, con for (const auto& entry : import_data.import_scripts) { if (!pwallet->HaveCScript(CScriptID(entry)) && !pwallet->AddCScript(entry)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error adding script to wallet"); + } + } + for (const auto& entry : privkey_map) { + const CKey& key = entry.second; + CPubKey pubkey = key.GetPubKey(); + const CKeyID& id = entry.first; + assert(key.VerifyPubKey(pubkey)); + pwallet->mapKeyMetadata[id].nCreateTime = timestamp; + // If the private key is not present in the wallet, insert it. + if (!pwallet->HaveKey(id) && !pwallet->AddKeyPubKey(key, pubkey)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); + } + pwallet->UpdateTimeFirstKey(timestamp); + } + for (const CKeyID& id : ordered_pubkeys) { + auto entry = pubkey_map.find(id); + if (entry == pubkey_map.end()) { + continue; + } + const CPubKey& pubkey = entry->second; + CPubKey temp; + if (!pwallet->GetPubKey(id, temp) && !pwallet->AddWatchOnly(GetScriptForRawPubKey(pubkey), timestamp)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + } + const auto& key_orig_it = import_data.key_origins.find(id); + if (key_orig_it != import_data.key_origins.end()) { + pwallet->AddKeyOrigin(pubkey, key_orig_it->second); } - } - for (const auto& entry : privkey_map) { - const CKey& key = entry.second; - CPubKey pubkey = key.GetPubKey(); - const CKeyID& id = entry.first; - assert(key.VerifyPubKey(pubkey)); pwallet->mapKeyMetadata[id].nCreateTime = timestamp; - // If the private key is not present in the wallet, insert it. - if (!pwallet->HaveKey(id) && !pwallet->AddKeyPubKey(key, pubkey)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding key to wallet"); + + // Add to keypool only works with pubkeys + if (add_keypool) { + pwallet->AddKeypoolPubkey(pubkey, internal); } - pwallet->UpdateTimeFirstKey(timestamp); } - for (const auto& entry : pubkey_map) { - const CPubKey& pubkey = entry.second; - const CKeyID& id = entry.first; - CPubKey temp; - if (!pwallet->GetPubKey(id, temp) && !pwallet->AddWatchOnly(GetScriptForRawPubKey(pubkey), timestamp)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + + for (const CScript& script : script_pub_keys) { + if (!have_solving_data || !::IsMine(*pwallet, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated + if (!pwallet->AddWatchOnly(script, timestamp)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + } } - } - if (!have_solving_data || !::IsMine(*pwallet, script)) { // Always call AddWatchOnly for non-solvable watch-only, so that watch timestamp gets updated - if (!pwallet->AddWatchOnly(script, timestamp)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error adding address to wallet"); + CTxDestination dest; + ExtractDestination(script, dest); + if (!internal && IsValidDestination(dest)) { + pwallet->SetAddressBook(dest, label, "receive"); } } - if (!internal) { - assert(IsValidDestination(dest)); - pwallet->SetAddressBook(dest, label, "receive"); - } result.pushKV("success", UniValue(true)); } catch (const UniValue& e) { @@ -1200,14 +1346,15 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported keys, addresses or scripts exists but related transactions are still missing.\n", { - {"requests", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "Data to be imported", + {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", { - {"", RPCArg::Type::OBJ, /* opt */ false, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"scriptPubKey", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "Type of scriptPubKey (string for script, json for address)", + {"desc", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Descriptor to import. If using descriptor, do not also provide address/scriptPubKey, scripts, or pubkeys"}, + {"scriptPubKey", RPCArg::Type::STR, RPCArg::Optional::NO, "Type of scriptPubKey (string for script, json for address). Should not be provided if using a descriptor", /* oneline_description */ "", {"\"<script>\" | { \"address\":\"<address>\" }", "string / json"} }, - {"timestamp", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "Creation time of the key in seconds since epoch (Jan 1 1970 GMT),\n" + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, "Creation time of the key in seconds since epoch (Jan 1 1970 GMT),\n" " or the string \"now\" to substitute the current synced blockchain time. The timestamp of the oldest\n" " key will determine how far back blockchain rescans need to begin for missing wallet transactions.\n" " \"now\" can be specified to bypass scanning, for keys which are known to never have been used, and\n" @@ -1215,28 +1362,30 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) " creation time of all keys being imported by the importmulti call will be scanned.", /* oneline_description */ "", {"timestamp | \"now\"", "integer / string"} }, - {"redeemscript", RPCArg::Type::STR, /* opt */ true, /* default_val */ "omitted", "Allowed only if the scriptPubKey is a P2SH or P2SH-P2WSH address/scriptPubKey"}, - {"witnessscript", RPCArg::Type::STR, /* opt */ true, /* default_val */ "omitted", "Allowed only if the scriptPubKey is a P2SH-P2WSH or P2WSH address/scriptPubKey"}, - {"pubkeys", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "empty array", "Array of strings giving pubkeys to import. They must occur in P2PKH or P2WPKH scripts. They are not required when the private key is also provided (see the \"keys\" argument).", + {"redeemscript", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Allowed only if the scriptPubKey is a P2SH or P2SH-P2WSH address/scriptPubKey"}, + {"witnessscript", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Allowed only if the scriptPubKey is a P2SH-P2WSH or P2WSH address/scriptPubKey"}, + {"pubkeys", RPCArg::Type::ARR, /* default */ "empty array", "Array of strings giving pubkeys to import. They must occur in P2PKH or P2WPKH scripts. They are not required when the private key is also provided (see the \"keys\" argument).", { - {"pubKey", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", ""}, + {"pubKey", RPCArg::Type::STR, RPCArg::Optional::OMITTED, ""}, } }, - {"keys", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "empty array", "Array of strings giving private keys to import. The corresponding public keys must occur in the output or redeemscript.", + {"keys", RPCArg::Type::ARR, /* default */ "empty array", "Array of strings giving private keys to import. The corresponding public keys must occur in the output or redeemscript.", { - {"key", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", ""}, + {"key", RPCArg::Type::STR, RPCArg::Optional::OMITTED, ""}, } }, - {"internal", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Stating whether matching outputs should be treated as not incoming payments (also known as change)"}, - {"watchonly", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Stating whether matching outputs should be considered watchonly."}, - {"label", RPCArg::Type::STR, /* opt */ true, /* default_val */ "''", "Label to assign to the address, only allowed with internal=false"}, + {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED, "If a ranged descriptor is used, this specifies the end or the range (in the form [begin,end]) to import"}, + {"internal", RPCArg::Type::BOOL, /* default */ "false", "Stating whether matching outputs should be treated as not incoming payments (also known as change)"}, + {"watchonly", RPCArg::Type::BOOL, /* default */ "false", "Stating whether matching outputs should be considered watchonly."}, + {"label", RPCArg::Type::STR, /* default */ "''", "Label to assign to the address, only allowed with internal=false"}, + {"keypool", RPCArg::Type::BOOL, /* default */ "false", "Stating whether imported public keys should be added to the keypool for when users request new addresses. Only allowed when wallet private keys are disabled"}, }, }, }, "\"requests\""}, - {"options", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "null", "", + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"rescan", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "Stating if should rescan the blockchain after all imports"}, + {"rescan", RPCArg::Type::BOOL, /* default */ "true", "Stating if should rescan the blockchain after all imports"}, }, "\"options\""}, }, diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e38ad3a0ba..31a3209a49 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -13,12 +13,12 @@ #include <validation.h> #include <key_io.h> #include <net.h> +#include <node/transaction.h> #include <outputtype.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/rbf.h> -#include <rpc/mining.h> #include <rpc/rawtransaction.h> #include <rpc/server.h> #include <rpc/util.h> @@ -26,10 +26,12 @@ #include <script/sign.h> #include <shutdown.h> #include <timedata.h> +#include <util/bip32.h> #include <util/system.h> #include <util/moneystr.h> #include <wallet/coincontrol.h> #include <wallet/feebumper.h> +#include <wallet/psbtwallet.h> #include <wallet/rpcwallet.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> @@ -121,8 +123,7 @@ static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& lo // Add opt-in RBF status std::string rbfStatus = "no"; if (confirms <= 0) { - LOCK(mempool.cs); - RBFTransactionState rbfState = IsRBFOptIn(*wtx.tx, mempool); + RBFTransactionState rbfState = chain.isRBFOptIn(*wtx.tx); if (rbfState == RBFTransactionState::UNKNOWN) rbfStatus = "unknown"; else if (rbfState == RBFTransactionState::REPLACEABLE_BIP125) @@ -158,8 +159,8 @@ static UniValue getnewaddress(const JSONRPCRequest& request) "If 'label' is specified, it is added to the address book \n" "so payments received with the address will be associated with 'label'.\n", { - {"label", RPCArg::Type::STR, /* opt */ true, /* default_val */ "null", "The label name for the address to be linked to. If not provided, the default label \"\" is used. It can also be set to the empty string \"\" to represent the default label. The label does not need to exist, it will be created if there is no label by the given name."}, - {"address_type", RPCArg::Type::STR, /* opt */ true, /* default_val */ "set by -addresstype", "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"label", RPCArg::Type::STR, /* default */ "\"\"", "The label name for the address to be linked to. It can also be set to the empty string \"\" to represent the default label. The label does not need to exist, it will be created if there is no label by the given name."}, + {"address_type", RPCArg::Type::STR, /* default */ "set by -addresstype", "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, }, RPCResult{ "\"address\" (string) The new bitcoin address\n" @@ -170,12 +171,12 @@ static UniValue getnewaddress(const JSONRPCRequest& request) }, }.ToString()); - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); - } - LOCK(pwallet->cs_wallet); + if (!pwallet->CanGetAddresses()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error: This wallet has no available keys"); + } + // Parse the label first so we don't generate a key if there's an error std::string label; if (!request.params[0].isNull()) @@ -220,7 +221,7 @@ static UniValue getrawchangeaddress(const JSONRPCRequest& request) "\nReturns a new Bitcoin address, for receiving change.\n" "This is for use with raw transactions, NOT normal use.\n", { - {"address_type", RPCArg::Type::STR, /* opt */ true, /* default_val */ "set by -changetype", "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"address_type", RPCArg::Type::STR, /* default */ "set by -changetype", "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, }, RPCResult{ "\"address\" (string) The address\n" @@ -231,12 +232,12 @@ static UniValue getrawchangeaddress(const JSONRPCRequest& request) }, }.ToString()); - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); - } - LOCK(pwallet->cs_wallet); + if (!pwallet->CanGetAddresses(true)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error: This wallet has no available keys"); + } + if (!pwallet->IsLocked()) { pwallet->TopUpKeyPool(); } @@ -276,8 +277,8 @@ static UniValue setlabel(const JSONRPCRequest& request) RPCHelpMan{"setlabel", "\nSets the label associated with the given address.\n", { - {"address", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The bitcoin address to be associated with a label."}, - {"label", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The label to assign to the address."}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to be associated with a label."}, + {"label", RPCArg::Type::STR, RPCArg::Optional::NO, "The label to assign to the address."}, }, RPCResults{}, RPCExamples{ @@ -316,7 +317,7 @@ static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet if (nValue > curBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Insufficient funds"); - if (pwallet->GetBroadcastTransactions() && !g_connman) { + if (pwallet->GetBroadcastTransactions() && !pwallet->chain().p2pEnabled()) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -338,7 +339,7 @@ static CTransactionRef SendMoney(interfaces::Chain::Lock& locked_chain, CWallet throw JSONRPCError(RPC_WALLET_ERROR, strError); } CValidationState state; - if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, reservekey, g_connman.get(), state)) { + if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, reservekey, state)) { strError = strprintf("Error: The transaction was rejected! Reason given: %s", FormatStateMessage(state)); throw JSONRPCError(RPC_WALLET_ERROR, strError); } @@ -360,18 +361,18 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) "\nSend an amount to a given address." + HelpRequiringPassphrase(pwallet) + "\n", { - {"address", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The bitcoin address to send to."}, - {"amount", RPCArg::Type::AMOUNT, /* opt */ false, /* default_val */ "", "The amount in " + CURRENCY_UNIT + " to send. eg 0.1"}, - {"comment", RPCArg::Type::STR, /* opt */ true, /* default_val */ "null", "A comment used to store what the transaction is for.\n" + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to send to."}, + {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount in " + CURRENCY_UNIT + " to send. eg 0.1"}, + {"comment", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A comment used to store what the transaction is for.\n" " This is not part of the transaction, just kept in your wallet."}, - {"comment_to", RPCArg::Type::STR, /* opt */ true, /* default_val */ "null", "A comment to store the name of the person or organization\n" + {"comment_to", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A comment to store the name of the person or organization\n" " to which you're sending the transaction. This is not part of the \n" " transaction, just kept in your wallet."}, - {"subtractfeefromamount", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "The fee will be deducted from the amount being sent.\n" + {"subtractfeefromamount", RPCArg::Type::BOOL, /* default */ "false", "The fee will be deducted from the amount being sent.\n" " The recipient will receive less bitcoins than you enter in the amount field."}, - {"replaceable", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "fallback to wallet's default", "Allow this transaction to be replaced by a transaction with higher fees via BIP 125"}, - {"conf_target", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "fallback to wallet's default", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* opt */ true, /* default_val */ "UNSET", "The fee estimate mode, must be one of:\n" + {"replaceable", RPCArg::Type::BOOL, /* default */ "fallback to wallet's default", "Allow this transaction to be replaced by a transaction with higher fees via BIP 125"}, + {"conf_target", RPCArg::Type::NUM, /* default */ "fallback to wallet's default", "Confirmation target (in blocks)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" " \"UNSET\"\n" " \"ECONOMICAL\"\n" " \"CONSERVATIVE\""}, @@ -422,7 +423,7 @@ static UniValue sendtoaddress(const JSONRPCRequest& request) } if (!request.params[6].isNull()) { - coin_control.m_confirm_target = ParseConfirmTarget(request.params[6]); + coin_control.m_confirm_target = ParseConfirmTarget(request.params[6], pwallet->chain().estimateMaxBlocks()); } if (!request.params[7].isNull()) { @@ -516,8 +517,8 @@ static UniValue signmessage(const JSONRPCRequest& request) "\nSign a message with the private key of an address" + HelpRequiringPassphrase(pwallet) + "\n", { - {"address", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The bitcoin address to use for the private key."}, - {"message", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The message to create a signature of."}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to use for the private key."}, + {"message", RPCArg::Type::STR, RPCArg::Optional::NO, "The message to create a signature of."}, }, RPCResult{ "\"signature\" (string) The signature of the message encoded in base 64\n" @@ -582,8 +583,8 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) RPCHelpMan{"getreceivedbyaddress", "\nReturns the total amount received by the given address in transactions with at least minconf confirmations.\n", { - {"address", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The bitcoin address for transactions."}, - {"minconf", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1", "Only include transactions confirmed at least this many times."}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address for transactions."}, + {"minconf", RPCArg::Type::NUM, /* default */ "1", "Only include transactions confirmed at least this many times."}, }, RPCResult{ "amount (numeric) The total amount in " + CURRENCY_UNIT + " received at this address.\n" @@ -604,7 +605,6 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -627,8 +627,9 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) CAmount nAmount = 0; for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) + if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { continue; + } for (const CTxOut& txout : wtx.tx->vout) if (txout.scriptPubKey == scriptPubKey) @@ -654,8 +655,8 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) RPCHelpMan{"getreceivedbylabel", "\nReturns the total amount received by addresses with <label> in transactions with at least [minconf] confirmations.\n", { - {"label", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The selected label, may be the default label using \"\"."}, - {"minconf", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1", "Only include transactions confirmed at least this many times."}, + {"label", RPCArg::Type::STR, RPCArg::Optional::NO, "The selected label, may be the default label using \"\"."}, + {"minconf", RPCArg::Type::NUM, /* default */ "1", "Only include transactions confirmed at least this many times."}, }, RPCResult{ "amount (numeric) The total amount in " + CURRENCY_UNIT + " received for this label.\n" @@ -676,7 +677,6 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -693,8 +693,9 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) CAmount nAmount = 0; for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) + if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { continue; + } for (const CTxOut& txout : wtx.tx->vout) { @@ -726,9 +727,9 @@ static UniValue getbalance(const JSONRPCRequest& request) "The available balance is what the wallet considers currently spendable, and is\n" "thus affected by options which limit spendability such as -spendzeroconfchange.\n", { - {"dummy", RPCArg::Type::STR, /* opt */ true, /* default_val */ "null", "Remains for backward compatibility. Must be excluded or set to \"*\"."}, - {"minconf", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "Only include transactions confirmed at least this many times."}, - {"include_watchonly", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Also include balance in watch-only addresses (see 'importaddress')"}, + {"dummy", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Remains for backward compatibility. Must be excluded or set to \"*\"."}, + {"minconf", RPCArg::Type::NUM, /* default */ "0", "Only include transactions confirmed at least this many times."}, + {"include_watchonly", RPCArg::Type::BOOL, /* default */ "false", "Also include balance in watch-only addresses (see 'importaddress')"}, }, RPCResult{ "amount (numeric) The total amount in " + CURRENCY_UNIT + " received for this wallet.\n" @@ -812,25 +813,25 @@ static UniValue sendmany(const JSONRPCRequest& request) "\nSend multiple times. Amounts are double-precision floating point numbers." + HelpRequiringPassphrase(pwallet) + "\n", { - {"dummy", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "Must be set to \"\" for backwards compatibility.", "\"\""}, - {"amounts", RPCArg::Type::OBJ, /* opt */ false, /* default_val */ "", "A json object with addresses and amounts", + {"dummy", RPCArg::Type::STR, RPCArg::Optional::NO, "Must be set to \"\" for backwards compatibility.", "\"\""}, + {"amounts", RPCArg::Type::OBJ, RPCArg::Optional::NO, "A json object with addresses and amounts", { - {"address", RPCArg::Type::AMOUNT, /* opt */ false, /* default_val */ "", "The bitcoin address is the key, the numeric amount (can be string) in " + CURRENCY_UNIT + " is the value"}, + {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The bitcoin address is the key, the numeric amount (can be string) in " + CURRENCY_UNIT + " is the value"}, }, }, - {"minconf", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1", "Only use the balance confirmed at least this many times."}, - {"comment", RPCArg::Type::STR, /* opt */ true, /* default_val */ "null", "A comment"}, - {"subtractfeefrom", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "null", "A json array with addresses.\n" + {"minconf", RPCArg::Type::NUM, /* default */ "1", "Only use the balance confirmed at least this many times."}, + {"comment", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A comment"}, + {"subtractfeefrom", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "A json array with addresses.\n" " The fee will be equally deducted from the amount of each selected address.\n" " Those recipients will receive less bitcoins than you enter in their corresponding amount field.\n" " If no addresses are specified here, the sender pays the fee.", { - {"address", RPCArg::Type::STR, /* opt */ true, /* default_val */ "", "Subtract fee from this address"}, + {"address", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Subtract fee from this address"}, }, }, - {"replaceable", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "fallback to wallet's default", "Allow this transaction to be replaced by a transaction with higher fees via BIP 125"}, - {"conf_target", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "fallback to wallet's default", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* opt */ true, /* default_val */ "UNSET", "The fee estimate mode, must be one of:\n" + {"replaceable", RPCArg::Type::BOOL, /* default */ "fallback to wallet's default", "Allow this transaction to be replaced by a transaction with higher fees via BIP 125"}, + {"conf_target", RPCArg::Type::NUM, /* default */ "fallback to wallet's default", "Confirmation target (in blocks)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" " \"UNSET\"\n" " \"ECONOMICAL\"\n" " \"CONSERVATIVE\""}, @@ -858,7 +859,7 @@ static UniValue sendmany(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - if (pwallet->GetBroadcastTransactions() && !g_connman) { + if (pwallet->GetBroadcastTransactions() && !pwallet->chain().p2pEnabled()) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); } @@ -884,7 +885,7 @@ static UniValue sendmany(const JSONRPCRequest& request) } if (!request.params[6].isNull()) { - coin_control.m_confirm_target = ParseConfirmTarget(request.params[6]); + coin_control.m_confirm_target = ParseConfirmTarget(request.params[6], pwallet->chain().estimateMaxBlocks()); } if (!request.params[7].isNull()) { @@ -946,7 +947,7 @@ static UniValue sendmany(const JSONRPCRequest& request) if (!fCreated) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, strFailReason); CValidationState state; - if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, keyChange, g_connman.get(), state)) { + if (!pwallet->CommitTransaction(tx, std::move(mapValue), {} /* orderForm */, keyChange, state)) { strFailReason = strprintf("Transaction commit failed:: %s", FormatStateMessage(state)); throw JSONRPCError(RPC_WALLET_ERROR, strFailReason); } @@ -972,14 +973,14 @@ static UniValue addmultisigaddress(const JSONRPCRequest& request) "See `importaddress` for watchonly p2sh address support.\n" "If 'label' is specified, assign address to that label.\n", { - {"nrequired", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The number of required signatures out of the n keys or addresses."}, - {"keys", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "A json array of bitcoin addresses or hex-encoded public keys", + {"nrequired", RPCArg::Type::NUM, RPCArg::Optional::NO, "The number of required signatures out of the n keys or addresses."}, + {"keys", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of bitcoin addresses or hex-encoded public keys", { - {"key", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "bitcoin address or hex-encoded public key"}, + {"key", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "bitcoin address or hex-encoded public key"}, }, }, - {"label", RPCArg::Type::STR, /* opt */ true, /* default_val */ "null", "A label to assign the addresses to."}, - {"address_type", RPCArg::Type::STR, /* opt */ true, /* default_val */ "set by -addresstype", "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"label", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "A label to assign the addresses to."}, + {"address_type", RPCArg::Type::STR, /* default */ "set by -addresstype", "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, }, RPCResult{ "{\n" @@ -1048,8 +1049,6 @@ struct tallyitem static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * const pwallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. - // Minimum confirmations int nMinDepth = 1; if (!params[0].isNull()) @@ -1080,8 +1079,9 @@ static UniValue ListReceived(interfaces::Chain::Lock& locked_chain, CWallet * co for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) + if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx)) { continue; + } int nDepth = wtx.GetDepthInMainChain(locked_chain); if (nDepth < nMinDepth) @@ -1206,10 +1206,10 @@ static UniValue listreceivedbyaddress(const JSONRPCRequest& request) RPCHelpMan{"listreceivedbyaddress", "\nList balances by receiving address.\n", { - {"minconf", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1", "The minimum number of confirmations before payments are included."}, - {"include_empty", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Whether to include addresses that haven't received any payments."}, - {"include_watchonly", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Whether to include watch-only addresses (see 'importaddress')."}, - {"address_filter", RPCArg::Type::STR, /* opt */ true, /* default_val */ "null", "If present, only return information on this address."}, + {"minconf", RPCArg::Type::NUM, /* default */ "1", "The minimum number of confirmations before payments are included."}, + {"include_empty", RPCArg::Type::BOOL, /* default */ "false", "Whether to include addresses that haven't received any payments."}, + {"include_watchonly", RPCArg::Type::BOOL, /* default */ "false", "Whether to include watch-only addresses (see 'importaddress')."}, + {"address_filter", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If present, only return information on this address."}, }, RPCResult{ "[\n" @@ -1259,9 +1259,9 @@ static UniValue listreceivedbylabel(const JSONRPCRequest& request) RPCHelpMan{"listreceivedbylabel", "\nList received transactions by label.\n", { - {"minconf", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1", "The minimum number of confirmations before payments are included."}, - {"include_empty", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Whether to include labels that haven't received any payments."}, - {"include_watchonly", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Whether to include watch-only addresses (see 'importaddress')."}, + {"minconf", RPCArg::Type::NUM, /* default */ "1", "The minimum number of confirmations before payments are included."}, + {"include_empty", RPCArg::Type::BOOL, /* default */ "false", "Whether to include labels that haven't received any payments."}, + {"include_watchonly", RPCArg::Type::BOOL, /* default */ "false", "Whether to include watch-only addresses (see 'importaddress')."}, }, RPCResult{ "[\n" @@ -1309,7 +1309,7 @@ static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) * @param filter_ismine The "is mine" filter flags. * @param filter_label Optional label string to filter incoming transactions. */ -static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* const pwallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) +static void ListTransactions(interfaces::Chain::Lock& locked_chain, CWallet* const pwallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { CAmount nFee; std::list<COutputEntry> listReceived; @@ -1400,11 +1400,11 @@ UniValue listtransactions(const JSONRPCRequest& request) "\nIf a label name is provided, this will return only incoming transactions paying to addresses with the specified label.\n" "\nReturns up to 'count' most recent transactions skipping the first 'from' transactions.\n", { - {"label", RPCArg::Type::STR, /* opt */ true, /* default_val */ "null", "If set, should be a valid label name to return only incoming transactions\n" + {"label", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If set, should be a valid label name to return only incoming transactions\n" " with the specified label, or \"*\" to disable filtering and return all transactions."}, - {"count", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "10", "The number of transactions to return"}, - {"skip", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "The number of transactions to skip"}, - {"include_watchonly", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Include transactions to watch-only addresses (see 'importaddress')"}, + {"count", RPCArg::Type::NUM, /* default */ "10", "The number of transactions to return"}, + {"skip", RPCArg::Type::NUM, /* default */ "0", "The number of transactions to skip"}, + {"include_watchonly", RPCArg::Type::BOOL, /* default */ "false", "Include transactions to watch-only addresses (see 'importaddress')"}, }, RPCResult{ "[\n" @@ -1535,10 +1535,10 @@ static UniValue listsinceblock(const JSONRPCRequest& request) "If \"blockhash\" is no longer a part of the main chain, transactions from the fork point onward are included.\n" "Additionally, if include_removed is set, transactions affecting the wallet which were removed are returned in the \"removed\" array.\n", { - {"blockhash", RPCArg::Type::STR, /* opt */ true, /* default_val */ "null", "If set, the block hash to list transactions since, otherwise list all transactions."}, - {"target_confirmations", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1", "Return the nth block hash from the main chain. e.g. 1 would mean the best block hash. Note: this is not used as a filter, but only affects [lastblock] in the return value"}, - {"include_watchonly", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Include transactions to watch-only addresses (see 'importaddress')"}, - {"include_removed", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "Show transactions that were removed due to a reorg in the \"removed\" array\n" + {"blockhash", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If set, the block hash to list transactions since, otherwise list all transactions."}, + {"target_confirmations", RPCArg::Type::NUM, /* default */ "1", "Return the nth block hash from the main chain. e.g. 1 would mean the best block hash. Note: this is not used as a filter, but only affects [lastblock] in the return value"}, + {"include_watchonly", RPCArg::Type::BOOL, /* default */ "false", "Include transactions to watch-only addresses (see 'importaddress')"}, + {"include_removed", RPCArg::Type::BOOL, /* default */ "true", "Show transactions that were removed due to a reorg in the \"removed\" array\n" " (not guaranteed to work on pruned nodes)"}, }, RPCResult{ @@ -1678,8 +1678,8 @@ static UniValue gettransaction(const JSONRPCRequest& request) RPCHelpMan{"gettransaction", "\nGet detailed information about in-wallet transaction <txid>\n", { - {"txid", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The transaction id"}, - {"include_watchonly", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Whether to include watch-only addresses in balance calculation and details[]"}, + {"txid", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction id"}, + {"include_watchonly", RPCArg::Type::BOOL, /* default */ "false", "Whether to include watch-only addresses in balance calculation and details[]"}, }, RPCResult{ "{\n" @@ -1784,7 +1784,7 @@ static UniValue abandontransaction(const JSONRPCRequest& request) "It only works on transactions which are not included in a block and are not currently in the mempool.\n" "It has no effect on transactions which are already abandoned.\n", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, }, RPCResults{}, RPCExamples{ @@ -1828,7 +1828,7 @@ static UniValue backupwallet(const JSONRPCRequest& request) RPCHelpMan{"backupwallet", "\nSafely copies current wallet file to destination, which can be a directory or a path with filename.\n", { - {"destination", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The destination directory or file"}, + {"destination", RPCArg::Type::STR, RPCArg::Optional::NO, "The destination directory or file"}, }, RPCResults{}, RPCExamples{ @@ -1868,7 +1868,7 @@ static UniValue keypoolrefill(const JSONRPCRequest& request) "\nFills the keypool."+ HelpRequiringPassphrase(pwallet) + "\n", { - {"newsize", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "100", "The new keypool size"}, + {"newsize", RPCArg::Type::NUM, /* default */ "100", "The new keypool size"}, }, RPCResults{}, RPCExamples{ @@ -1921,8 +1921,8 @@ static UniValue walletpassphrase(const JSONRPCRequest& request) "Issuing the walletpassphrase command while the wallet is already unlocked will set a new unlock\n" "time that overrides the old one.\n", { - {"passphrase", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The wallet passphrase"}, - {"timeout", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The time to keep the decryption key in seconds; capped at 100000000 (~3 years)."}, + {"passphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet passphrase"}, + {"timeout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The time to keep the decryption key in seconds; capped at 100000000 (~3 years)."}, }, RPCResults{}, RPCExamples{ @@ -2004,8 +2004,8 @@ static UniValue walletpassphrasechange(const JSONRPCRequest& request) RPCHelpMan{"walletpassphrasechange", "\nChanges the wallet passphrase from 'oldpassphrase' to 'newpassphrase'.\n", { - {"oldpassphrase", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The current passphrase"}, - {"newpassphrase", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The new passphrase"}, + {"oldpassphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The current passphrase"}, + {"newpassphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The new passphrase"}, }, RPCResults{}, RPCExamples{ @@ -2106,7 +2106,7 @@ static UniValue encryptwallet(const JSONRPCRequest& request) "Use the walletpassphrase call for this, and then walletlock call.\n" "If the wallet is already encrypted, use the walletpassphrasechange call.\n", { - {"passphrase", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The pass phrase to encrypt the wallet with. It must be at least 1 character, but should be long."}, + {"passphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The pass phrase to encrypt the wallet with. It must be at least 1 character, but should be long."}, }, RPCResults{}, RPCExamples{ @@ -2168,13 +2168,13 @@ static UniValue lockunspent(const JSONRPCRequest& request) "is always cleared (by virtue of process exit) when a node stops or fails.\n" "Also see the listunspent call\n", { - {"unlock", RPCArg::Type::BOOL, /* opt */ false, /* default_val */ "", "Whether to unlock (true) or lock (false) the specified transactions"}, - {"transactions", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "empty array", "A json array of objects. Each object the txid (string) vout (numeric).", + {"unlock", RPCArg::Type::BOOL, RPCArg::Optional::NO, "Whether to unlock (true) or lock (false) the specified transactions"}, + {"transactions", RPCArg::Type::ARR, /* default */ "empty array", "A json array of objects. Each object the txid (string) vout (numeric).", { - {"", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id"}, - {"vout", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The output number"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, }, }, }, @@ -2348,7 +2348,7 @@ static UniValue settxfee(const JSONRPCRequest& request) RPCHelpMan{"settxfee", "\nSet the transaction fee per kB for this wallet. Overrides the global -paytxfee command line parameter.\n", { - {"amount", RPCArg::Type::AMOUNT, /* opt */ false, /* default_val */ "", "The transaction fee in " + CURRENCY_UNIT + "/kB"}, + {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The transaction fee in " + CURRENCY_UNIT + "/kB"}, }, RPCResult{ "true|false (boolean) Returns true if successful\n" @@ -2405,7 +2405,6 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) " \"unlocked_until\": ttt, (numeric) the timestamp in seconds since epoch (midnight Jan 1 1970 GMT) that the wallet is unlocked for transfers, or 0 if the wallet is locked\n" " \"paytxfee\": x.xxxx, (numeric) the transaction fee configuration, set in " + CURRENCY_UNIT + "/kB\n" " \"hdseedid\": \"<hash160>\" (string, optional) the Hash160 of the HD seed (only present when HD is enabled)\n" - " \"hdmasterkeyid\": \"<hash160>\" (string, optional) alias for hdseedid retained for backwards-compatibility. Will be removed in V0.18.\n" " \"private_keys_enabled\": true|false (boolean) false if privatekeys are disabled for this wallet (enforced watch-only wallet)\n" "}\n" }, @@ -2434,7 +2433,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) obj.pushKV("keypoololdest", pwallet->GetOldestKeyPoolTime()); obj.pushKV("keypoolsize", (int64_t)kpExternalSize); CKeyID seed_id = pwallet->GetHDChain().seed_id; - if (!seed_id.IsNull() && pwallet->CanSupportFeature(FEATURE_HD_SPLIT)) { + if (pwallet->CanSupportFeature(FEATURE_HD_SPLIT)) { obj.pushKV("keypoolsize_hd_internal", (int64_t)(pwallet->GetKeyPoolSize() - kpExternalSize)); } if (pwallet->IsCrypted()) { @@ -2443,7 +2442,6 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) obj.pushKV("paytxfee", ValueFromAmount(pwallet->m_pay_tx_fee.GetFeePerK())); if (!seed_id.IsNull()) { obj.pushKV("hdseedid", seed_id.GetHex()); - obj.pushKV("hdmasterkeyid", seed_id.GetHex()); } obj.pushKV("private_keys_enabled", !pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); return obj; @@ -2529,7 +2527,7 @@ static UniValue loadwallet(const JSONRPCRequest& request) "\nNote that all wallet command-line options used when starting bitcoind will be" "\napplied to the new wallet (eg -zapwallettxes, upgradewallet, rescan, etc).\n", { - {"filename", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The wallet directory or .dat file."}, + {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet directory or .dat file."}, }, RPCResult{ "{\n" @@ -2544,7 +2542,6 @@ static UniValue loadwallet(const JSONRPCRequest& request) }.ToString()); WalletLocation location(request.params[0].get_str()); - std::string error; if (!location.Exists()) { throw JSONRPCError(RPC_WALLET_NOT_FOUND, "Wallet " + location.GetName() + " not found."); @@ -2556,18 +2553,9 @@ static UniValue loadwallet(const JSONRPCRequest& request) } } - std::string warning; - if (!CWallet::Verify(*g_rpc_interfaces->chain, location, false, error, warning)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Wallet file verification failed: " + error); - } - - std::shared_ptr<CWallet> const wallet = CWallet::CreateWalletFromFile(*g_rpc_interfaces->chain, location); - if (!wallet) { - throw JSONRPCError(RPC_WALLET_ERROR, "Wallet loading failed."); - } - AddWallet(wallet); - - wallet->postInitProcess(); + std::string error, warning; + std::shared_ptr<CWallet> const wallet = LoadWallet(*g_rpc_interfaces->chain, location, error, warning); + if (!wallet) throw JSONRPCError(RPC_WALLET_ERROR, error); UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); @@ -2578,13 +2566,14 @@ static UniValue loadwallet(const JSONRPCRequest& request) static UniValue createwallet(const JSONRPCRequest& request) { - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { + if (request.fHelp || request.params.size() < 1 || request.params.size() > 3) { throw std::runtime_error( RPCHelpMan{"createwallet", "\nCreates and loads a new wallet.\n", { - {"wallet_name", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The name for the new wallet. If this is a path, the wallet will be created at the path location."}, - {"disable_private_keys", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Disable the possibility of private keys (only watchonlys are possible in this mode)."}, + {"wallet_name", RPCArg::Type::STR, RPCArg::Optional::NO, "The name for the new wallet. If this is a path, the wallet will be created at the path location."}, + {"disable_private_keys", RPCArg::Type::BOOL, /* default */ "false", "Disable the possibility of private keys (only watchonlys are possible in this mode)."}, + {"blank", RPCArg::Type::BOOL, /* default */ "false", "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, }, RPCResult{ "{\n" @@ -2601,9 +2590,13 @@ static UniValue createwallet(const JSONRPCRequest& request) std::string error; std::string warning; - bool disable_privatekeys = false; - if (!request.params[1].isNull()) { - disable_privatekeys = request.params[1].get_bool(); + uint64_t flags = 0; + if (!request.params[1].isNull() && request.params[1].get_bool()) { + flags |= WALLET_FLAG_DISABLE_PRIVATE_KEYS; + } + + if (!request.params[2].isNull() && request.params[2].get_bool()) { + flags |= WALLET_FLAG_BLANK_WALLET; } WalletLocation location(request.params[0].get_str()); @@ -2616,7 +2609,7 @@ static UniValue createwallet(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Wallet file verification failed: " + error); } - std::shared_ptr<CWallet> const wallet = CWallet::CreateWalletFromFile(*g_rpc_interfaces->chain, location, (disable_privatekeys ? (uint64_t)WALLET_FLAG_DISABLE_PRIVATE_KEYS : 0)); + std::shared_ptr<CWallet> const wallet = CWallet::CreateWalletFromFile(*g_rpc_interfaces->chain, location, flags); if (!wallet) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet creation failed."); } @@ -2639,7 +2632,7 @@ static UniValue unloadwallet(const JSONRPCRequest& request) "Unloads the wallet referenced by the request endpoint otherwise unloads the wallet specified in the argument.\n" "Specifying the wallet name on a wallet endpoint is invalid.", { - {"wallet_name", RPCArg::Type::STR, /* opt */ true, /* default_val */ "the wallet name from the RPC request", "The name of the wallet to unload."}, + {"wallet_name", RPCArg::Type::STR, /* default */ "the wallet name from the RPC request", "The name of the wallet to unload."}, }, RPCResults{}, RPCExamples{ @@ -2699,8 +2692,9 @@ static UniValue resendwallettransactions(const JSONRPCRequest& request) }.ToString() ); - if (!g_connman) + if (!pwallet->chain().p2pEnabled()) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); + } auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); @@ -2709,7 +2703,7 @@ static UniValue resendwallettransactions(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Error: Wallet transaction broadcasting is disabled with -walletbroadcast"); } - std::vector<uint256> txids = pwallet->ResendWalletTransactionsBefore(*locked_chain, GetTime(), g_connman.get()); + std::vector<uint256> txids = pwallet->ResendWalletTransactionsBefore(*locked_chain, GetTime()); UniValue result(UniValue::VARR); for (const uint256& txid : txids) { @@ -2734,21 +2728,21 @@ static UniValue listunspent(const JSONRPCRequest& request) "with between minconf and maxconf (inclusive) confirmations.\n" "Optionally filter to only include txouts paid to specified addresses.\n", { - {"minconf", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1", "The minimum confirmations to filter"}, - {"maxconf", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "9999999", "The maximum confirmations to filter"}, - {"addresses", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "empty array", "A json array of bitcoin addresses to filter", + {"minconf", RPCArg::Type::NUM, /* default */ "1", "The minimum confirmations to filter"}, + {"maxconf", RPCArg::Type::NUM, /* default */ "9999999", "The maximum confirmations to filter"}, + {"addresses", RPCArg::Type::ARR, /* default */ "empty array", "A json array of bitcoin addresses to filter", { - {"address", RPCArg::Type::STR, /* opt */ true, /* default_val */ "", "bitcoin address"}, + {"address", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "bitcoin address"}, }, }, - {"include_unsafe", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "Include outputs that are not safe to spend\n" + {"include_unsafe", RPCArg::Type::BOOL, /* default */ "true", "Include outputs that are not safe to spend\n" " See description of \"safe\" attribute below."}, - {"query_options", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "null", "JSON with query options", + {"query_options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "JSON with query options", { - {"minimumAmount", RPCArg::Type::AMOUNT, /* opt */ true, /* default_val */ "0", "Minimum value of each UTXO in " + CURRENCY_UNIT + ""}, - {"maximumAmount", RPCArg::Type::AMOUNT, /* opt */ true, /* default_val */ "unlimited", "Maximum value of each UTXO in " + CURRENCY_UNIT + ""}, - {"maximumCount", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "unlimited", "Maximum number of UTXOs"}, - {"minimumSumAmount", RPCArg::Type::AMOUNT, /* opt */ true, /* default_val */ "unlimited", "Minimum sum value of all UTXOs in " + CURRENCY_UNIT + ""}, + {"minimumAmount", RPCArg::Type::AMOUNT, /* default */ "0", "Minimum value of each UTXO in " + CURRENCY_UNIT + ""}, + {"maximumAmount", RPCArg::Type::AMOUNT, /* default */ "unlimited", "Maximum value of each UTXO in " + CURRENCY_UNIT + ""}, + {"maximumCount", RPCArg::Type::NUM, /* default */ "unlimited", "Maximum number of UTXOs"}, + {"minimumSumAmount", RPCArg::Type::AMOUNT, /* default */ "unlimited", "Minimum sum value of all UTXOs in " + CURRENCY_UNIT + ""}, }, "query_options"}, }, @@ -2762,7 +2756,8 @@ static UniValue listunspent(const JSONRPCRequest& request) " \"scriptPubKey\" : \"key\", (string) the script key\n" " \"amount\" : x.xxx, (numeric) the transaction output amount in " + CURRENCY_UNIT + "\n" " \"confirmations\" : n, (numeric) The number of confirmations\n" - " \"redeemScript\" : n (string) The redeemScript if scriptPubKey is P2SH\n" + " \"redeemScript\" : \"script\" (string) The redeemScript if scriptPubKey is P2SH\n" + " \"witnessScript\" : \"script\" (string) witnessScript if the scriptPubKey is P2WSH or P2SH-P2WSH\n" " \"spendable\" : xxx, (bool) Whether we have the private keys to spend this output\n" " \"solvable\" : xxx, (bool) Whether we know how to spend this output, ignoring the lack of keys\n" " \"desc\" : xxx, (string, only when solvable) A descriptor for spending this output\n" @@ -2876,6 +2871,28 @@ static UniValue listunspent(const JSONRPCRequest& request) CScript redeemScript; if (pwallet->GetCScript(hash, redeemScript)) { entry.pushKV("redeemScript", HexStr(redeemScript.begin(), redeemScript.end())); + // Now check if the redeemScript is actually a P2WSH script + CTxDestination witness_destination; + if (redeemScript.IsPayToWitnessScriptHash()) { + bool extracted = ExtractDestination(redeemScript, witness_destination); + assert(extracted); + // Also return the witness script + const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(witness_destination); + CScriptID id; + CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); + CScript witnessScript; + if (pwallet->GetCScript(id, witnessScript)) { + entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); + } + } + } + } else if (scriptPubKey.IsPayToWitnessScriptHash()) { + const WitnessV0ScriptHash& whash = boost::get<WitnessV0ScriptHash>(address); + CScriptID id; + CRIPEMD160().Write(whash.begin(), whash.size()).Finalize(id.begin()); + CScript witnessScript; + if (pwallet->GetCScript(id, witnessScript)) { + entry.pushKV("witnessScript", HexStr(witnessScript.begin(), witnessScript.end())); } } } @@ -2975,7 +2992,7 @@ void FundTransaction(CWallet* const pwallet, CMutableTransaction& tx, CAmount& f if (options.exists("feeRate")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both conf_target and feeRate"); } - coinControl.m_confirm_target = ParseConfirmTarget(options["conf_target"]); + coinControl.m_confirm_target = ParseConfirmTarget(options["conf_target"], pwallet->chain().estimateMaxBlocks()); } if (options.exists("estimate_mode")) { if (options.exists("feeRate")) { @@ -3036,33 +3053,33 @@ static UniValue fundrawtransaction(const JSONRPCRequest& request) "You can see whether this is the case by checking the \"solvable\" field in the listunspent output.\n" "Only pay-to-pubkey, multisig, and P2SH versions thereof are currently supported for watch-only\n", { - {"hexstring", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The hex string of the raw transaction"}, - {"options", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "null", "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", + {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", { - {"changeAddress", RPCArg::Type::STR, /* opt */ true, /* default_val */ "pool address", "The bitcoin address to receive the change"}, - {"changePosition", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "random", "The index of the change output"}, - {"change_type", RPCArg::Type::STR, /* opt */ true, /* default_val */ "set by -changetype", "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, - {"includeWatching", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Also select inputs which are watch only"}, - {"lockUnspents", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Lock selected unspent outputs"}, - {"feeRate", RPCArg::Type::AMOUNT, /* opt */ true, /* default_val */ "not set: makes wallet determine the fee", "Set a specific fee rate in " + CURRENCY_UNIT + "/kB"}, - {"subtractFeeFromOutputs", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "empty array", "A json array of integers.\n" + {"changeAddress", RPCArg::Type::STR, /* default */ "pool address", "The bitcoin address to receive the change"}, + {"changePosition", RPCArg::Type::NUM, /* default */ "random", "The index of the change output"}, + {"change_type", RPCArg::Type::STR, /* default */ "set by -changetype", "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"includeWatching", RPCArg::Type::BOOL, /* default */ "false", "Also select inputs which are watch only"}, + {"lockUnspents", RPCArg::Type::BOOL, /* default */ "false", "Lock selected unspent outputs"}, + {"feeRate", RPCArg::Type::AMOUNT, /* default */ "not set: makes wallet determine the fee", "Set a specific fee rate in " + CURRENCY_UNIT + "/kB"}, + {"subtractFeeFromOutputs", RPCArg::Type::ARR, /* default */ "empty array", "A json array of integers.\n" " The fee will be equally deducted from the amount of each specified output.\n" " Those recipients will receive less bitcoins than you enter in their corresponding amount field.\n" " If no outputs are specified here, the sender pays the fee.", { - {"vout_index", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "", "The zero-based output index, before a change output is added."}, + {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "fallback to wallet's default", "Marks this transaction as BIP125 replaceable.\n" + {"replaceable", RPCArg::Type::BOOL, /* default */ "fallback to wallet's default", "Marks this transaction as BIP125 replaceable.\n" " Allows this transaction to be replaced by a transaction with higher fees"}, - {"conf_target", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "fallback to wallet's default", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* opt */ true, /* default_val */ "UNSET", "The fee estimate mode, must be one of:\n" + {"conf_target", RPCArg::Type::NUM, /* default */ "fallback to wallet's default", "Confirmation target (in blocks)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" " \"UNSET\"\n" " \"ECONOMICAL\"\n" " \"CONSERVATIVE\""}, }, "options"}, - {"iswitness", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "depends on heuristic tests", "Whether the transaction hex is a serialized witness transaction \n" + {"iswitness", RPCArg::Type::BOOL, /* default */ "depends on heuristic tests", "Whether the transaction hex is a serialized witness transaction \n" " If iswitness is not present, heuristic tests will be used in decoding"}, }, RPCResult{ @@ -3123,21 +3140,22 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) "this transaction depends on but may not yet be in the block chain." + HelpRequiringPassphrase(pwallet) + "\n", { - {"hexstring", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The transaction hex string"}, - {"prevtxs", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "null", "A json array of previous dependent transaction outputs", + {"hexstring", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction hex string"}, + {"prevtxs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "A json array of previous dependent transaction outputs", { - {"", RPCArg::Type::OBJ, /* opt */ false, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id"}, - {"vout", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The output number"}, - {"scriptPubKey", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "script key"}, - {"redeemScript", RPCArg::Type::STR_HEX, /* opt */ true, /* default_val */ "omitted", "(required for P2SH or P2WSH)"}, - {"amount", RPCArg::Type::AMOUNT, /* opt */ false, /* default_val */ "", "The amount spent"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, + {"scriptPubKey", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "script key"}, + {"redeemScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2SH) redeem script"}, + {"witnessScript", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "(required for P2WSH or P2SH-P2WSH) witness script"}, + {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The amount spent"}, }, }, }, }, - {"sighashtype", RPCArg::Type::STR, /* opt */ true, /* default_val */ "ALL", "The signature hash type. Must be one of\n" + {"sighashtype", RPCArg::Type::STR, /* default */ "ALL", "The signature hash type. Must be one of\n" " \"ALL\"\n" " \"NONE\"\n" " \"SINGLE\"\n" @@ -3206,22 +3224,22 @@ static UniValue bumpfee(const JSONRPCRequest& request) "At a minimum, the new fee rate must be high enough to pay an additional new relay fee (incrementalfee\n" "returned by getnetworkinfo) to enter the node's mempool.\n", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The txid to be bumped"}, - {"options", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "null", "", + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The txid to be bumped"}, + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"confTarget", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "fallback to wallet's default", "Confirmation target (in blocks)"}, - {"totalFee", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "fallback to 'confTarget'", "Total fee (NOT feerate) to pay, in satoshis.\n" + {"confTarget", RPCArg::Type::NUM, /* default */ "fallback to wallet's default", "Confirmation target (in blocks)"}, + {"totalFee", RPCArg::Type::NUM, /* default */ "fallback to 'confTarget'", "Total fee (NOT feerate) to pay, in satoshis.\n" " In rare cases, the actual fee paid might be slightly higher than the specified\n" " totalFee if the tx change output has to be removed because it is too close to\n" " the dust threshold."}, - {"replaceable", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "Whether the new transaction should still be\n" + {"replaceable", RPCArg::Type::BOOL, /* default */ "true", "Whether the new transaction should still be\n" " marked bip-125 replaceable. If true, the sequence numbers in the transaction will\n" " be left unchanged from the original. If false, any input sequence numbers in the\n" " original transaction that were less than 0xfffffffe will be increased to 0xfffffffe\n" " so the new transaction will not be explicitly bip-125 replaceable (though it may\n" " still be replaceable in practice, for example if it has unconfirmed ancestors which\n" " are replaceable)."}, - {"estimate_mode", RPCArg::Type::STR, /* opt */ true, /* default_val */ "UNSET", "The fee estimate mode, must be one of:\n" + {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" " \"UNSET\"\n" " \"ECONOMICAL\"\n" " \"CONSERVATIVE\""}, @@ -3264,7 +3282,7 @@ static UniValue bumpfee(const JSONRPCRequest& request) if (options.exists("confTarget") && options.exists("totalFee")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "confTarget and totalFee options should not both be set. Please provide either a confirmation target for fee estimation or an explicit total fee for the transaction."); } else if (options.exists("confTarget")) { // TODO: alias this to conf_target - coin_control.m_confirm_target = ParseConfirmTarget(options["confTarget"]); + coin_control.m_confirm_target = ParseConfirmTarget(options["confTarget"], pwallet->chain().estimateMaxBlocks()); } else if (options.exists("totalFee")) { totalFee = options["totalFee"].get_int64(); if (totalFee <= 0) { @@ -3338,62 +3356,6 @@ static UniValue bumpfee(const JSONRPCRequest& request) return result; } -UniValue generate(const JSONRPCRequest& request) -{ - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - CWallet* const pwallet = wallet.get(); - - - if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { - return NullUniValue; - } - - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { - throw std::runtime_error( - RPCHelpMan{"generate", - "\nMine up to nblocks blocks immediately (before the RPC call returns) to an address in the wallet.\n", - { - {"nblocks", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "How many blocks are generated immediately."}, - {"maxtries", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "1000000", "How many iterations to try."}, - }, - RPCResult{ - "[ blockhashes ] (array) hashes of blocks generated\n" - }, - RPCExamples{ - "\nGenerate 11 blocks\n" - + HelpExampleCli("generate", "11") - }, - }.ToString()); - } - - if (!IsDeprecatedRPCEnabled("generate")) { - throw JSONRPCError(RPC_METHOD_DEPRECATED, "The wallet generate rpc method is deprecated and will be fully removed in v0.19. " - "To use generate in v0.18, restart bitcoind with -deprecatedrpc=generate.\n" - "Clients should transition to using the node rpc method generatetoaddress\n"); - } - - int num_generate = request.params[0].get_int(); - uint64_t max_tries = 1000000; - if (!request.params[1].isNull()) { - max_tries = request.params[1].get_int(); - } - - std::shared_ptr<CReserveScript> coinbase_script; - pwallet->GetScriptForMining(coinbase_script); - - // If the keypool is exhausted, no script is returned at all. Catch this. - if (!coinbase_script) { - throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, "Error: Keypool ran out, please call keypoolrefill first"); - } - - //throw an error if no script was provided - if (coinbase_script->reserveScript.empty()) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "No coinbase script available"); - } - - return generateBlocks(coinbase_script, num_generate, max_tries, true); -} - UniValue rescanblockchain(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -3408,8 +3370,8 @@ UniValue rescanblockchain(const JSONRPCRequest& request) RPCHelpMan{"rescanblockchain", "\nRescan the local blockchain for wallet related transactions.\n", { - {"start_height", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "block height where the rescan should start"}, - {"stop_height", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "", "the last block height that should be scanned. If none is provided it will rescan up to the tip at return time of this call."}, + {"start_height", RPCArg::Type::NUM, /* default */ "0", "block height where the rescan should start"}, + {"stop_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "the last block height that should be scanned. If none is provided it will rescan up to the tip at return time of this call."}, }, RPCResult{ "{\n" @@ -3622,7 +3584,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) "\nReturn information about the given bitcoin address. Some information requires the address\n" "to be in the wallet.\n", { - {"address", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The bitcoin address to get the information of."}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to get the information of."}, }, RPCResult{ "{\n" @@ -3652,7 +3614,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) " \"timestamp\" : timestamp, (number, optional) The creation time of the key if available in seconds since epoch (Jan 1 1970 GMT)\n" " \"hdkeypath\" : \"keypath\" (string, optional) The HD keypath if the key is HD and available\n" " \"hdseedid\" : \"<hash160>\" (string, optional) The Hash160 of the HD seed\n" - " \"hdmasterkeyid\" : \"<hash160>\" (string, optional) alias for hdseedid maintained for backwards compatibility. Will be removed in V0.18.\n" + " \"hdmasterfingerprint\" : \"<hash160>\" (string, optional) The fingperint of the master key.\n" " \"labels\" (object) Array of labels associated with the address.\n" " [\n" " { (json object of label data)\n" @@ -3715,10 +3677,10 @@ UniValue getaddressinfo(const JSONRPCRequest& request) } if (meta) { ret.pushKV("timestamp", meta->nCreateTime); - if (!meta->hdKeypath.empty()) { - ret.pushKV("hdkeypath", meta->hdKeypath); + if (meta->has_key_origin) { + ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); - ret.pushKV("hdmasterkeyid", meta->hd_seed_id.GetHex()); + ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint, meta->key_origin.fingerprint + 4)); } } @@ -3749,7 +3711,7 @@ static UniValue getaddressesbylabel(const JSONRPCRequest& request) RPCHelpMan{"getaddressesbylabel", "\nReturns the list of addresses assigned the specified label.\n", { - {"label", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The label."}, + {"label", RPCArg::Type::STR, RPCArg::Optional::NO, "The label."}, }, RPCResult{ "{ (json object with addresses as keys)\n" @@ -3797,7 +3759,7 @@ static UniValue listlabels(const JSONRPCRequest& request) RPCHelpMan{"listlabels", "\nReturns the list of all labels, or labels that are assigned to addresses with a specific purpose.\n", { - {"purpose", RPCArg::Type::STR, /* opt */ true, /* default_val */ "null", "Address purpose to list labels for ('send','receive'). An empty string is the same as not providing this argument."}, + {"purpose", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Address purpose to list labels for ('send','receive'). An empty string is the same as not providing this argument."}, }, RPCResult{ "[ (json array of string)\n" @@ -3857,11 +3819,11 @@ UniValue sethdseed(const JSONRPCRequest& request) "\nNote that you will need to MAKE A NEW BACKUP of your wallet after setting the HD wallet seed." + HelpRequiringPassphrase(pwallet) + "\n", { - {"newkeypool", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "Whether to flush old unused addresses, including change addresses, from the keypool and regenerate it.\n" + {"newkeypool", RPCArg::Type::BOOL, /* default */ "true", "Whether to flush old unused addresses, including change addresses, from the keypool and regenerate it.\n" " If true, the next address from getnewaddress and change address from getrawchangeaddress will be from this new seed.\n" " If false, addresses (including change addresses if the wallet already had HD Chain Split enabled) from the existing\n" " keypool will be used until it has been depleted."}, - {"seed", RPCArg::Type::STR, /* opt */ true, /* default_val */ "random seed", "The WIF private key to use as the new HD seed.\n" + {"seed", RPCArg::Type::STR, /* default */ "random seed", "The WIF private key to use as the new HD seed.\n" " The seed value can be retrieved using the dumpwallet command. It is the private key marked hdseed=1"}, }, RPCResults{}, @@ -3874,7 +3836,7 @@ UniValue sethdseed(const JSONRPCRequest& request) }.ToString()); } - if (IsInitialBlockDownload()) { + if (pwallet->chain().isInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Cannot set a new HD seed while still in Initial Block Download"); } @@ -3886,7 +3848,7 @@ UniValue sethdseed(const JSONRPCRequest& request) LOCK(pwallet->cs_wallet); // Do not do anything to non-HD wallets - if (!pwallet->IsHDEnabled()) { + if (!pwallet->CanSupportFeature(FEATURE_HD)) { throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Start with -upgradewallet in order to upgrade a non-HD wallet to HD"); } @@ -3932,60 +3894,6 @@ void AddKeypathToMap(const CWallet* pwallet, const CKeyID& keyID, std::map<CPubK hd_keypaths.emplace(vchPubKey, std::move(info)); } -bool FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& psbtx, int sighash_type, bool sign, bool bip32derivs) -{ - LOCK(pwallet->cs_wallet); - // Get all of the previous transactions - bool complete = true; - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - const CTxIn& txin = psbtx.tx->vin[i]; - PSBTInput& input = psbtx.inputs.at(i); - - if (PSBTInputSigned(input)) { - continue; - } - - // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. - if (!input.IsSane()) { - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "PSBT input is not sane."); - } - - // If we have no utxo, grab it from the wallet. - if (!input.non_witness_utxo && input.witness_utxo.IsNull()) { - const uint256& txhash = txin.prevout.hash; - const auto it = pwallet->mapWallet.find(txhash); - if (it != pwallet->mapWallet.end()) { - const CWalletTx& wtx = it->second; - // We only need the non_witness_utxo, which is a superset of the witness_utxo. - // The signing code will switch to the smaller witness_utxo if this is ok. - input.non_witness_utxo = wtx.tx; - } - } - - // Get the Sighash type - if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Specified Sighash and sighash in PSBT do not match."); - } - - complete &= SignPSBTInput(HidingSigningProvider(pwallet, !sign, !bip32derivs), psbtx, i, sighash_type); - } - - // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change - for (unsigned int i = 0; i < psbtx.tx->vout.size(); ++i) { - const CTxOut& out = psbtx.tx->vout.at(i); - PSBTOutput& psbt_out = psbtx.outputs.at(i); - - // Fill a SignatureData with output info - SignatureData sigdata; - psbt_out.FillSignatureData(sigdata); - - MutableTransactionSignatureCreator creator(psbtx.tx.get_ptr(), 0, out.nValue, 1); - ProduceSignature(HidingSigningProvider(pwallet, true, !bip32derivs), creator, out.scriptPubKey, sigdata); - psbt_out.FromSignatureData(sigdata); - } - return complete; -} - UniValue walletprocesspsbt(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -4002,16 +3910,16 @@ UniValue walletprocesspsbt(const JSONRPCRequest& request) "that we can sign for." + HelpRequiringPassphrase(pwallet) + "\n", { - {"psbt", RPCArg::Type::STR, /* opt */ false, /* default_val */ "", "The transaction base64 string"}, - {"sign", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "true", "Also sign the transaction when updating"}, - {"sighashtype", RPCArg::Type::STR, /* opt */ true, /* default_val */ "ALL", "The signature hash type to sign with if not specified by the PSBT. Must be one of\n" + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction base64 string"}, + {"sign", RPCArg::Type::BOOL, /* default */ "true", "Also sign the transaction when updating"}, + {"sighashtype", RPCArg::Type::STR, /* default */ "ALL", "The signature hash type to sign with if not specified by the PSBT. Must be one of\n" " \"ALL\"\n" " \"NONE\"\n" " \"SINGLE\"\n" " \"ALL|ANYONECANPAY\"\n" " \"NONE|ANYONECANPAY\"\n" " \"SINGLE|ANYONECANPAY\""}, - {"bip32derivs", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "If true, includes the BIP 32 derivation paths for public keys if we know them"}, + {"bip32derivs", RPCArg::Type::BOOL, /* default */ "false", "If true, includes the BIP 32 derivation paths for public keys if we know them"}, }, RPCResult{ "{\n" @@ -4030,7 +3938,7 @@ UniValue walletprocesspsbt(const JSONRPCRequest& request) // Unserialize the transaction PartiallySignedTransaction psbtx; std::string error; - if (!DecodePSBT(psbtx, request.params[0].get_str(), error)) { + if (!DecodeBase64PSBT(psbtx, request.params[0].get_str(), error)) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("TX decode failed %s", error)); } @@ -4040,7 +3948,11 @@ UniValue walletprocesspsbt(const JSONRPCRequest& request) // Fill transaction with our data and also sign bool sign = request.params[1].isNull() ? true : request.params[1].get_bool(); bool bip32derivs = request.params[3].isNull() ? false : request.params[3].get_bool(); - bool complete = FillPSBT(pwallet, psbtx, nHashType, sign, bip32derivs); + bool complete = true; + const TransactionError err = FillPSBT(pwallet, psbtx, complete, nHashType, sign, bip32derivs); + if (err != TransactionError::OK) { + throw JSONRPCTransactionError(err); + } UniValue result(UniValue::VOBJ); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); @@ -4066,61 +3978,61 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) "\nCreates and funds a transaction in the Partially Signed Transaction format. Inputs will be added if supplied inputs are not enough\n" "Implements the Creator and Updater roles.\n", { - {"inputs", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "A json array of json objects", + {"inputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "A json array of json objects", { - {"", RPCArg::Type::OBJ, /* opt */ false, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"txid", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "The transaction id"}, - {"vout", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The output number"}, - {"sequence", RPCArg::Type::NUM, /* opt */ false, /* default_val */ "", "The sequence number"}, + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, + {"sequence", RPCArg::Type::NUM, RPCArg::Optional::NO, "The sequence number"}, }, }, }, }, - {"outputs", RPCArg::Type::ARR, /* opt */ false, /* default_val */ "", "a json array with outputs (key-value pairs), where none of the keys are duplicated.\n" + {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "a json array with outputs (key-value pairs), where none of the keys are duplicated.\n" "That is, each address can only appear once and there can only be one 'data' object.\n" "For compatibility reasons, a dictionary, which holds the key-value pairs directly, is also\n" " accepted as second parameter.", { - {"", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"address", RPCArg::Type::AMOUNT, /* opt */ false, /* default_val */ "", "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT + ""}, + {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT + ""}, }, }, - {"", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "", "", + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", { - {"data", RPCArg::Type::STR_HEX, /* opt */ false, /* default_val */ "", "A key-value pair. The key must be \"data\", the value is hex-encoded data"}, + {"data", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "A key-value pair. The key must be \"data\", the value is hex-encoded data"}, }, }, }, }, - {"locktime", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "0", "Raw locktime. Non-0 value also locktime-activates inputs"}, - {"options", RPCArg::Type::OBJ, /* opt */ true, /* default_val */ "null", "", + {"locktime", RPCArg::Type::NUM, /* default */ "0", "Raw locktime. Non-0 value also locktime-activates inputs"}, + {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"changeAddress", RPCArg::Type::STR_HEX, /* opt */ true, /* default_val */ "pool address", "The bitcoin address to receive the change"}, - {"changePosition", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "random", "The index of the change output"}, - {"change_type", RPCArg::Type::STR, /* opt */ true, /* default_val */ "set by -changetype", "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, - {"includeWatching", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Also select inputs which are watch only"}, - {"lockUnspents", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Lock selected unspent outputs"}, - {"feeRate", RPCArg::Type::AMOUNT, /* opt */ true, /* default_val */ "not set: makes wallet determine the fee", "Set a specific fee rate in " + CURRENCY_UNIT + "/kB"}, - {"subtractFeeFromOutputs", RPCArg::Type::ARR, /* opt */ true, /* default_val */ "empty array", "A json array of integers.\n" + {"changeAddress", RPCArg::Type::STR_HEX, /* default */ "pool address", "The bitcoin address to receive the change"}, + {"changePosition", RPCArg::Type::NUM, /* default */ "random", "The index of the change output"}, + {"change_type", RPCArg::Type::STR, /* default */ "set by -changetype", "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"includeWatching", RPCArg::Type::BOOL, /* default */ "false", "Also select inputs which are watch only"}, + {"lockUnspents", RPCArg::Type::BOOL, /* default */ "false", "Lock selected unspent outputs"}, + {"feeRate", RPCArg::Type::AMOUNT, /* default */ "not set: makes wallet determine the fee", "Set a specific fee rate in " + CURRENCY_UNIT + "/kB"}, + {"subtractFeeFromOutputs", RPCArg::Type::ARR, /* default */ "empty array", "A json array of integers.\n" " The fee will be equally deducted from the amount of each specified output.\n" " Those recipients will receive less bitcoins than you enter in their corresponding amount field.\n" " If no outputs are specified here, the sender pays the fee.", { - {"vout_index", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "", "The zero-based output index, before a change output is added."}, + {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "Marks this transaction as BIP125 replaceable.\n" + {"replaceable", RPCArg::Type::BOOL, /* default */ "false", "Marks this transaction as BIP125 replaceable.\n" " Allows this transaction to be replaced by a transaction with higher fees"}, - {"conf_target", RPCArg::Type::NUM, /* opt */ true, /* default_val */ "Fallback to wallet's confirmation target", "Confirmation target (in blocks)"}, - {"estimate_mode", RPCArg::Type::STR, /* opt */ true, /* default_val */ "UNSET", "The fee estimate mode, must be one of:\n" + {"conf_target", RPCArg::Type::NUM, /* default */ "Fallback to wallet's confirmation target", "Confirmation target (in blocks)"}, + {"estimate_mode", RPCArg::Type::STR, /* default */ "UNSET", "The fee estimate mode, must be one of:\n" " \"UNSET\"\n" " \"ECONOMICAL\"\n" " \"CONSERVATIVE\""}, }, "options"}, - {"bip32derivs", RPCArg::Type::BOOL, /* opt */ true, /* default_val */ "false", "If true, includes the BIP 32 derivation paths for public keys if we know them"}, + {"bip32derivs", RPCArg::Type::BOOL, /* default */ "false", "If true, includes the BIP 32 derivation paths for public keys if we know them"}, }, RPCResult{ "{\n" @@ -4154,7 +4066,11 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) // Fill transaction with out data but don't sign bool bip32derivs = request.params[4].isNull() ? false : request.params[4].get_bool(); - FillPSBT(pwallet, psbtx, 1, false, bip32derivs); + bool complete = true; + const TransactionError err = FillPSBT(pwallet, psbtx, complete, 1, false, bip32derivs); + if (err != TransactionError::OK) { + throw JSONRPCTransactionError(err); + } // Serialize the PSBT CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); @@ -4182,7 +4098,6 @@ UniValue importmulti(const JSONRPCRequest& request); static const CRPCCommand commands[] = { // category name actor (function) argNames // --------------------- ------------------------ ----------------------- ---------- - { "generating", "generate", &generate, {"nblocks","maxtries"} }, { "hidden", "resendwallettransactions", &resendwallettransactions, {} }, { "rawtransactions", "fundrawtransaction", &fundrawtransaction, {"hexstring","options","iswitness"} }, { "wallet", "abandontransaction", &abandontransaction, {"txid"} }, @@ -4190,7 +4105,7 @@ static const CRPCCommand commands[] = { "wallet", "addmultisigaddress", &addmultisigaddress, {"nrequired","keys","label","address_type"} }, { "wallet", "backupwallet", &backupwallet, {"destination"} }, { "wallet", "bumpfee", &bumpfee, {"txid", "options"} }, - { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys"} }, + { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank"} }, { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, { "wallet", "dumpwallet", &dumpwallet, {"filename"} }, { "wallet", "encryptwallet", &encryptwallet, {"passphrase"} }, diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index abd7750874..58053bde59 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -30,5 +30,4 @@ bool EnsureWalletIsAvailable(CWallet *, bool avoidException); UniValue getaddressinfo(const JSONRPCRequest& request); UniValue signrawtransactionwithwallet(const JSONRPCRequest& request); -bool FillPSBT(const CWallet* pwallet, PartiallySignedTransaction& psbtx, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false); #endif //BITCOIN_WALLET_RPCWALLET_H diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 9918eeb89f..789e86e21b 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -4,7 +4,9 @@ #include <key_io.h> #include <script/sign.h> +#include <util/bip32.h> #include <util/strencodings.h> +#include <wallet/psbtwallet.h> #include <wallet/rpcwallet.h> #include <wallet/wallet.h> #include <univalue.h> @@ -60,7 +62,8 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) ssData >> psbtx; // Fill transaction with our data - FillPSBT(&m_wallet, psbtx, SIGHASH_ALL, false, true); + bool complete = true; + BOOST_REQUIRE_EQUAL(TransactionError::OK, FillPSBT(&m_wallet, psbtx, complete, SIGHASH_ALL, false, true)); // Get the final tx CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index fcb34c3706..af57dbf5f6 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -44,6 +44,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = chainActive.Tip(); + LockAnnotation lock(::cs_main); auto locked_chain = chain->lock(); // Verify ScanForWalletTransactions accommodates a null start block. @@ -123,6 +124,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); CBlockIndex* newTip = chainActive.Tip(); + LockAnnotation lock(::cs_main); auto locked_chain = chain->lock(); // Prune the older block file. @@ -268,6 +270,7 @@ static int64_t AddTx(CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64 SetMockTime(mockTime); CBlockIndex* block = nullptr; if (blockTime > 0) { + LockAnnotation lock(::cs_main); auto locked_chain = wallet.chain().lock(); auto inserted = mapBlockIndex.emplace(GetRandHash(), new CBlockIndex); assert(inserted.second); @@ -365,7 +368,7 @@ public: CCoinControl dummy; BOOST_CHECK(wallet->CreateTransaction(*m_locked_chain, {recipient}, tx, reservekey, fee, changePos, error, dummy)); CValidationState state; - BOOST_CHECK(wallet->CommitTransaction(tx, {}, {}, reservekey, nullptr, state)); + BOOST_CHECK(wallet->CommitTransaction(tx, {}, {}, reservekey, state)); CMutableTransaction blocktx; { LOCK(wallet->cs_wallet); @@ -449,6 +452,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { auto chain = interfaces::MakeChain(); std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(*chain, WalletLocation(), WalletDatabase::CreateDummy()); + wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); BOOST_CHECK(!wallet->TopUpKeyPool(1000)); CPubKey pubkey; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index bdddcd718b..e47f5fa956 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -12,6 +12,7 @@ #include <consensus/validation.h> #include <fs.h> #include <interfaces/chain.h> +#include <interfaces/wallet.h> #include <key.h> #include <key_io.h> #include <keystore.h> @@ -27,6 +28,7 @@ #include <shutdown.h> #include <timedata.h> #include <txmempool.h> +#include <util/bip32.h> #include <util/moneystr.h> #include <wallet/fees.h> @@ -130,6 +132,28 @@ void UnloadWallet(std::shared_ptr<CWallet>&& wallet) } } +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::string& warning) +{ + if (!CWallet::Verify(chain, location, false, error, warning)) { + error = "Wallet file verification failed: " + error; + return nullptr; + } + + std::shared_ptr<CWallet> wallet = CWallet::CreateWalletFromFile(chain, location); + if (!wallet) { + error = "Wallet loading failed."; + return nullptr; + } + AddWallet(wallet); + wallet->postInitProcess(); + return wallet; +} + +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::string& error, std::string& warning) +{ + return LoadWallet(chain, WalletLocation(name), error, warning); +} + const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; const uint256 CMerkleTx::ABANDON_HASH(uint256S("0000000000000000000000000000000000000000000000000000000000000001")); @@ -168,7 +192,8 @@ const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const CPubKey CWallet::GenerateNewKey(WalletBatch &batch, bool internal) { assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); - AssertLockHeld(cs_wallet); // mapKeyMetadata + assert(!IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); + AssertLockHeld(cs_wallet); bool fCompressed = CanSupportFeature(FEATURE_COMPRPUBKEY); // default to compressed public keys if we want 0.6.0 wallets CKey secret; @@ -177,7 +202,7 @@ CPubKey CWallet::GenerateNewKey(WalletBatch &batch, bool internal) int64_t nCreationTime = GetTime(); CKeyMetadata metadata(nCreationTime); - // use HD key derivation if HD was enabled during wallet creation + // use HD key derivation if HD was enabled during wallet creation and a seed is present if (IsHDEnabled()) { DeriveNewChildKey(batch, metadata, secret, (CanSupportFeature(FEATURE_HD_SPLIT) ? internal : false)); } else { @@ -232,24 +257,33 @@ void CWallet::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey if (internal) { chainChildKey.Derive(childKey, hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); metadata.hdKeypath = "m/0'/1'/" + std::to_string(hdChain.nInternalChainCounter) + "'"; + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(1 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(hdChain.nInternalChainCounter | BIP32_HARDENED_KEY_LIMIT); hdChain.nInternalChainCounter++; } else { chainChildKey.Derive(childKey, hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); metadata.hdKeypath = "m/0'/0'/" + std::to_string(hdChain.nExternalChainCounter) + "'"; + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(0 | BIP32_HARDENED_KEY_LIMIT); + metadata.key_origin.path.push_back(hdChain.nExternalChainCounter | BIP32_HARDENED_KEY_LIMIT); hdChain.nExternalChainCounter++; } } while (HaveKey(childKey.key.GetPubKey().GetID())); secret = childKey.key; metadata.hd_seed_id = hdChain.seed_id; + CKeyID master_id = masterKey.key.GetPubKey().GetID(); + std::copy(master_id.begin(), master_id.begin() + 4, metadata.key_origin.fingerprint); + metadata.has_key_origin = true; // update the chain model in the database if (!batch.WriteHDChain(hdChain)) throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); } -bool CWallet::AddKeyPubKeyWithDB(WalletBatch &batch, const CKey& secret, const CPubKey &pubkey) +bool CWallet::AddKeyPubKeyWithDB(WalletBatch& batch, const CKey& secret, const CPubKey& pubkey) { - AssertLockHeld(cs_wallet); // mapKeyMetadata + AssertLockHeld(cs_wallet); // Make sure we aren't adding private keys to private key disabled wallets assert(!IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); @@ -283,6 +317,7 @@ bool CWallet::AddKeyPubKeyWithDB(WalletBatch &batch, const CKey& secret, const C secret.GetPrivKey(), mapKeyMetadata[pubkey.GetID()]); } + UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); return true; } @@ -310,20 +345,68 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, } } -void CWallet::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &meta) +void CWallet::LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata& meta) { - AssertLockHeld(cs_wallet); // mapKeyMetadata + AssertLockHeld(cs_wallet); UpdateTimeFirstKey(meta.nCreateTime); mapKeyMetadata[keyID] = meta; } -void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &meta) +void CWallet::LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata& meta) { - AssertLockHeld(cs_wallet); // m_script_metadata + AssertLockHeld(cs_wallet); UpdateTimeFirstKey(meta.nCreateTime); m_script_metadata[script_id] = meta; } +// Writes a keymetadata for a public key. overwrite specifies whether to overwrite an existing metadata for that key if there exists one. +bool CWallet::WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, const bool overwrite) +{ + return WalletBatch(*database).WriteKeyMetadata(meta, pubkey, overwrite); +} + +void CWallet::UpgradeKeyMetadata() +{ + AssertLockHeld(cs_wallet); + if (IsLocked() || IsWalletFlagSet(WALLET_FLAG_KEY_ORIGIN_METADATA)) { + return; + } + + std::unique_ptr<WalletBatch> batch = MakeUnique<WalletBatch>(*database); + size_t cnt = 0; + for (auto& meta_pair : mapKeyMetadata) { + CKeyMetadata& meta = meta_pair.second; + if (!meta.hd_seed_id.IsNull() && !meta.has_key_origin && meta.hdKeypath != "s") { // If the hdKeypath is "s", that's the seed and it doesn't have a key origin + CKey key; + GetKey(meta.hd_seed_id, key); + CExtKey masterKey; + masterKey.SetSeed(key.begin(), key.size()); + // Add to map + CKeyID master_id = masterKey.key.GetPubKey().GetID(); + std::copy(master_id.begin(), master_id.begin() + 4, meta.key_origin.fingerprint); + if (!ParseHDKeypath(meta.hdKeypath, meta.key_origin.path)) { + throw std::runtime_error("Invalid stored hdKeypath"); + } + meta.has_key_origin = true; + if (meta.nVersion < CKeyMetadata::VERSION_WITH_KEY_ORIGIN) { + meta.nVersion = CKeyMetadata::VERSION_WITH_KEY_ORIGIN; + } + + // Write meta to wallet + CPubKey pubkey; + if (GetPubKey(meta_pair.first, pubkey)) { + batch->WriteKeyMetadata(meta, pubkey, true); + if (++cnt % 1000 == 0) { + // avoid creating overlarge in-memory batches in case the wallet contains large amounts of keys + batch.reset(new WalletBatch(*database)); + } + } + } + } + batch.reset(); //write before setting the flag + SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); +} + bool CWallet::LoadCryptedKey(const CPubKey &vchPubKey, const std::vector<unsigned char> &vchCryptedSecret) { return CCryptoKeyStore::AddCryptedKey(vchPubKey, vchCryptedSecret); @@ -349,7 +432,11 @@ bool CWallet::AddCScript(const CScript& redeemScript) { if (!CCryptoKeyStore::AddCScript(redeemScript)) return false; - return WalletBatch(*database).WriteCScript(Hash160(redeemScript), redeemScript); + if (WalletBatch(*database).WriteCScript(Hash160(redeemScript), redeemScript)) { + UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); + return true; + } + return false; } bool CWallet::LoadCScript(const CScript& redeemScript) @@ -374,7 +461,11 @@ bool CWallet::AddWatchOnly(const CScript& dest) const CKeyMetadata& meta = m_script_metadata[CScriptID(dest)]; UpdateTimeFirstKey(meta.nCreateTime); NotifyWatchonlyChanged(true); - return WalletBatch(*database).WriteWatchOnly(dest, meta); + if (WalletBatch(*database).WriteWatchOnly(dest, meta)) { + UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); + return true; + } + return false; } bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) @@ -414,8 +505,11 @@ bool CWallet::Unlock(const SecureString& strWalletPassphrase, bool accept_no_key return false; if (!crypter.Decrypt(pMasterKey.second.vchCryptedKey, _vMasterKey)) continue; // try another master key - if (CCryptoKeyStore::Unlock(_vMasterKey, accept_no_keys)) + if (CCryptoKeyStore::Unlock(_vMasterKey, accept_no_keys)) { + // Now that we've unlocked, upgrade the key metadata + UpgradeKeyMetadata(); return true; + } } } return false; @@ -475,7 +569,7 @@ void CWallet::ChainStateFlushed(const CBlockLocator& loc) void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in, bool fExplicit) { - LOCK(cs_wallet); // nWalletVersion + LOCK(cs_wallet); if (nWalletVersion >= nVersion) return; @@ -499,7 +593,7 @@ void CWallet::SetMinVersion(enum WalletFeature nVersion, WalletBatch* batch_in, bool CWallet::SetMaxVersion(int nVersion) { - LOCK(cs_wallet); // nWalletVersion, nWalletMaxVersion + LOCK(cs_wallet); // cannot downgrade below current version if (nWalletVersion > nVersion) return false; @@ -783,9 +877,9 @@ DBErrors CWallet::ReorderTransactions() return DBErrors::LOAD_OK; } -int64_t CWallet::IncOrderPosNext(WalletBatch *batch) +int64_t CWallet::IncOrderPosNext(WalletBatch* batch) { - AssertLockHeld(cs_wallet); // nOrderPosNext + AssertLockHeld(cs_wallet); int64_t nRet = nOrderPosNext++; if (batch) { batch->WriteOrderPosNext(nOrderPosNext); @@ -847,7 +941,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) wtx.BindWallet(this); bool fInsertedNew = ret.second; if (fInsertedNew) { - wtx.nTimeReceived = GetAdjustedTime(); + wtx.nTimeReceived = chain().getAdjustedTime(); wtx.nOrderPos = IncOrderPosNext(&batch); wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); wtx.nTimeSmart = ComputeTimeSmart(wtx); @@ -1375,6 +1469,7 @@ CPubKey CWallet::DeriveNewSeed(const CKey& key) // set the hd keypath to "s" -> Seed, refers the seed to itself metadata.hdKeypath = "s"; + metadata.has_key_origin = false; metadata.hd_seed_id = seed.GetID(); { @@ -1402,6 +1497,7 @@ void CWallet::SetHDSeed(const CPubKey& seed) newHdChain.seed_id = seed.GetID(); SetHDChain(newHdChain, false); NotifyCanGetAddressesChanged(); + UnsetWalletFlag(WALLET_FLAG_BLANK_WALLET); } void CWallet::SetHDChain(const CHDChain& chain, bool memonly) @@ -1418,6 +1514,30 @@ bool CWallet::IsHDEnabled() const return !hdChain.seed_id.IsNull(); } +bool CWallet::CanGenerateKeys() +{ + // A wallet can generate keys if it has an HD seed (IsHDEnabled) or it is a non-HD wallet (pre FEATURE_HD) + LOCK(cs_wallet); + return IsHDEnabled() || !CanSupportFeature(FEATURE_HD); +} + +bool CWallet::CanGetAddresses(bool internal) +{ + LOCK(cs_wallet); + // Check if the keypool has keys + bool keypool_has_keys; + if (internal && CanSupportFeature(FEATURE_HD_SPLIT)) { + keypool_has_keys = setInternalKeyPool.size() > 0; + } else { + keypool_has_keys = KeypoolCountExternalKeys() > 0; + } + // If the keypool doesn't have keys, check if we can generate them + if (!keypool_has_keys) { + return CanGenerateKeys(); + } + return keypool_has_keys; +} + void CWallet::SetWalletFlag(uint64_t flags) { LOCK(cs_wallet); @@ -1426,6 +1546,14 @@ void CWallet::SetWalletFlag(uint64_t flags) throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); } +void CWallet::UnsetWalletFlag(uint64_t flag) +{ + LOCK(cs_wallet); + m_wallet_flags &= ~flag; + if (!WalletBatch(*database).WriteWalletFlags(m_wallet_flags)) + throw std::runtime_error(std::string(__func__) + ": writing wallet flags failed"); +} + bool CWallet::IsWalletFlagSet(uint64_t flag) { return (m_wallet_flags & flag); @@ -1759,25 +1887,21 @@ void CWallet::ReacceptWalletTransactions() for (const std::pair<const int64_t, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *(item.second); CValidationState state; - wtx.AcceptToMemoryPool(*locked_chain, maxTxFee, state); + wtx.AcceptToMemoryPool(*locked_chain, state); } } -bool CWalletTx::RelayWalletTransaction(interfaces::Chain::Lock& locked_chain, CConnman* connman) +bool CWalletTx::RelayWalletTransaction(interfaces::Chain::Lock& locked_chain) { assert(pwallet->GetBroadcastTransactions()); if (!IsCoinBase() && !isAbandoned() && GetDepthInMainChain(locked_chain) == 0) { CValidationState state; /* GetDepthInMainChain already catches known conflicts. */ - if (InMempool() || AcceptToMemoryPool(locked_chain, maxTxFee, state)) { + if (InMempool() || AcceptToMemoryPool(locked_chain, state)) { pwallet->WalletLogPrintf("Relaying wtx %s\n", GetHash().ToString()); - if (connman) { - CInv inv(MSG_TX, GetHash()); - connman->ForEachNode([&inv](CNode* pnode) - { - pnode->PushInventory(inv); - }); + if (pwallet->chain().p2pEnabled()) { + pwallet->chain().relayTransaction(GetHash()); return true; } } @@ -1948,11 +2072,10 @@ bool CWalletTx::InMempool() const bool CWalletTx::IsTrusted(interfaces::Chain::Lock& locked_chain) const { - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. - // Quick answer in most cases - if (!CheckFinalTx(*tx)) + if (!locked_chain.checkFinalTx(*tx)) { return false; + } int nDepth = GetDepthInMainChain(locked_chain); if (nDepth >= 1) return true; @@ -1988,7 +2111,7 @@ bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const return CTransaction(tx1) == CTransaction(tx2); } -std::vector<uint256> CWallet::ResendWalletTransactionsBefore(interfaces::Chain::Lock& locked_chain, int64_t nTime, CConnman* connman) +std::vector<uint256> CWallet::ResendWalletTransactionsBefore(interfaces::Chain::Lock& locked_chain, int64_t nTime) { std::vector<uint256> result; @@ -2007,8 +2130,9 @@ std::vector<uint256> CWallet::ResendWalletTransactionsBefore(interfaces::Chain:: for (const std::pair<const unsigned int, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *item.second; - if (wtx.RelayWalletTransaction(locked_chain, connman)) + if (wtx.RelayWalletTransaction(locked_chain)) { result.push_back(wtx.GetHash()); + } } return result; } @@ -2032,7 +2156,7 @@ void CWallet::ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman // Rebroadcast unconfirmed txes older than 5 minutes before the last // block was found: auto locked_chain = chain().assumeLocked(); // Temporary. Removed in upcoming lock cleanup - std::vector<uint256> relayed = ResendWalletTransactionsBefore(*locked_chain, nBestBlockTime-5*60, connman); + std::vector<uint256> relayed = ResendWalletTransactionsBefore(*locked_chain, nBestBlockTime-5*60); if (!relayed.empty()) WalletLogPrintf("%s: rebroadcast %u unconfirmed transactions\n", __func__, relayed.size()); } @@ -2136,7 +2260,6 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const // trusted. CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth) const { - LockAnnotation lock(::cs_main); // Temporary, for CheckFinalTx below. Removed in upcoming commit. auto locked_chain = chain().lock(); LOCK(cs_wallet); @@ -2144,7 +2267,7 @@ CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth) cons for (const auto& entry : mapWallet) { const CWalletTx& wtx = entry.second; const int depth = wtx.GetDepthInMainChain(*locked_chain); - if (depth < 0 || !CheckFinalTx(*wtx.tx) || wtx.IsImmatureCoinBase(*locked_chain)) { + if (depth < 0 || !locked_chain->checkFinalTx(*wtx.tx) || wtx.IsImmatureCoinBase(*locked_chain)) { continue; } @@ -2198,8 +2321,9 @@ void CWallet::AvailableCoins(interfaces::Chain::Lock& locked_chain, std::vector< const uint256& wtxid = entry.first; const CWalletTx* pcoin = &entry.second; - if (!CheckFinalTx(*pcoin->tx)) + if (!locked_chain.checkFinalTx(*pcoin->tx)) { continue; + } if (pcoin->IsImmatureCoinBase(locked_chain)) continue; @@ -2361,10 +2485,10 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const CoinEligibil FeeCalculation feeCalc; CCoinControl temp; temp.m_confirm_target = 1008; - CFeeRate long_term_feerate = GetMinimumFeeRate(*this, temp, ::mempool, ::feeEstimator, &feeCalc); + CFeeRate long_term_feerate = GetMinimumFeeRate(*this, temp, &feeCalc); // Calculate cost of change - CAmount cost_of_change = GetDiscardRate(*this, ::feeEstimator).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); + CAmount cost_of_change = GetDiscardRate(*this).GetFee(coin_selection_params.change_spend_size) + coin_selection_params.effective_fee.GetFee(coin_selection_params.change_output_size); // Filter by the min conf specs and add to utxo_pool and calculate effective value for (OutputGroup& group : groups) { @@ -2490,9 +2614,9 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm return res; } -bool CWallet::SignTransaction(CMutableTransaction &tx) +bool CWallet::SignTransaction(CMutableTransaction& tx) { - AssertLockHeld(cs_wallet); // mapWallet + AssertLockHeld(cs_wallet); // sign the new tx int nIn = 0; @@ -2713,8 +2837,8 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // post-backup change. // Reserve a new key pair from key pool - if (IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - strFailReason = _("Can't generate a change-address key. Private keys are disabled for this wallet."); + if (!CanGetAddresses(true)) { + strFailReason = _("Can't generate a change-address key. No keys in the internal keypool and can't generate any keys."); return false; } CPubKey vchPubKey; @@ -2734,10 +2858,10 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std CTxOut change_prototype_txout(0, scriptChange); coin_selection_params.change_output_size = GetSerializeSize(change_prototype_txout); - CFeeRate discard_rate = GetDiscardRate(*this, ::feeEstimator); + CFeeRate discard_rate = GetDiscardRate(*this); // Get the fee rate to use effective values in coin selection - CFeeRate nFeeRateNeeded = GetMinimumFeeRate(*this, coin_control, ::mempool, ::feeEstimator, &feeCalc); + CFeeRate nFeeRateNeeded = GetMinimumFeeRate(*this, coin_control, &feeCalc); nFeeRet = 0; bool pick_new_inputs = true; @@ -2870,7 +2994,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std return false; } - nFeeNeeded = GetMinimumFee(*this, nBytes, coin_control, ::mempool, ::feeEstimator, &feeCalc); + nFeeNeeded = GetMinimumFee(*this, nBytes, coin_control, &feeCalc); if (feeCalc.reason == FeeReason::FALLBACK && !m_allow_fallback_fee) { // eventually allow a fallback fee strFailReason = _("Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee."); @@ -2898,7 +3022,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // change output. Only try this once. if (nChangePosInOut == -1 && nSubtractFeeFromAmount == 0 && pick_new_inputs) { unsigned int tx_size_with_change = nBytes + coin_selection_params.change_output_size + 2; // Add 2 as a buffer in case increasing # of outputs changes compact size - CAmount fee_needed_with_change = GetMinimumFee(*this, tx_size_with_change, coin_control, ::mempool, ::feeEstimator, nullptr); + CAmount fee_needed_with_change = GetMinimumFee(*this, tx_size_with_change, coin_control, nullptr); CAmount minimum_value_for_change = GetDustThreshold(change_prototype_txout, discard_rate); if (nFeeRet >= fee_needed_with_change + minimum_value_for_change) { pick_new_inputs = false; @@ -3003,16 +3127,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std if (gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS)) { // Lastly, ensure this tx will pass the mempool's chain limits - LockPoints lp; - CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); - CTxMemPool::setEntries setAncestors; - size_t nLimitAncestors = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); - size_t nLimitAncestorSize = gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000; - size_t nLimitDescendants = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); - size_t nLimitDescendantSize = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000; - std::string errString; - LOCK(::mempool.cs); - if (!::mempool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) { + if (!chain().checkChainLimits(tx)) { strFailReason = _("Transaction has too long of a mempool chain"); return false; } @@ -3032,7 +3147,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std /** * Call after CreateTransaction unless you want to abort */ -bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CConnman* connman, CValidationState& state) +bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CValidationState& state) { { auto locked_chain = chain().lock(); @@ -3069,11 +3184,11 @@ bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve if (fBroadcastTransactions) { // Broadcast - if (!wtx.AcceptToMemoryPool(*locked_chain, maxTxFee, state)) { + if (!wtx.AcceptToMemoryPool(*locked_chain, state)) { WalletLogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", FormatStateMessage(state)); // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. } else { - wtx.RelayWalletTransaction(*locked_chain, connman); + wtx.RelayWalletTransaction(*locked_chain); } } } @@ -3103,7 +3218,8 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { LOCK(cs_KeyStore); // This wallet is in its first run if all of these are empty - fFirstRunRet = mapKeys.empty() && mapCryptedKeys.empty() && mapWatchKeys.empty() && setWatchOnly.empty() && mapScripts.empty() && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + fFirstRunRet = mapKeys.empty() && mapCryptedKeys.empty() && mapWatchKeys.empty() && setWatchOnly.empty() && mapScripts.empty() + && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET); } if (nLoadWalletRet != DBErrors::LOAD_OK) @@ -3114,8 +3230,8 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) { - AssertLockHeld(cs_wallet); // mapWallet - DBErrors nZapSelectTxRet = WalletBatch(*database,"cr+").ZapSelectTx(vHashIn, vHashOut); + AssertLockHeld(cs_wallet); + DBErrors nZapSelectTxRet = WalletBatch(*database, "cr+").ZapSelectTx(vHashIn, vHashOut); for (uint256 hash : vHashOut) { const auto& it = mapWallet.find(hash); wtxOrdered.erase(it->second.m_it_wtxOrdered); @@ -3141,7 +3257,6 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 MarkDirty(); return DBErrors::LOAD_OK; - } DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) @@ -3172,7 +3287,7 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s { bool fUpdated = false; { - LOCK(cs_wallet); // mapAddressBook + LOCK(cs_wallet); std::map<CTxDestination, CAddressBookData>::iterator mi = mapAddressBook.find(address); fUpdated = mi != mapAddressBook.end(); mapAddressBook[address].name = strName; @@ -3189,7 +3304,7 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s bool CWallet::DelAddressBook(const CTxDestination& address) { { - LOCK(cs_wallet); // mapAddressBook + LOCK(cs_wallet); // Delete destdata tuples associated with address std::string strAddress = EncodeDestination(address); @@ -3261,7 +3376,7 @@ bool CWallet::NewKeyPool() size_t CWallet::KeypoolCountExternalKeys() { - AssertLockHeld(cs_wallet); // setExternalKeyPool + AssertLockHeld(cs_wallet); return setExternalKeyPool.size() + set_pre_split_keypool.size(); } @@ -3288,7 +3403,7 @@ void CWallet::LoadKeyPool(int64_t nIndex, const CKeyPool &keypool) bool CWallet::TopUpKeyPool(unsigned int kpSize) { - if (IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (!CanGenerateKeys()) { return false; } { @@ -3322,20 +3437,8 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) internal = true; } - assert(m_max_keypool_index < std::numeric_limits<int64_t>::max()); // How in the hell did you use so many keys? - int64_t index = ++m_max_keypool_index; - CPubKey pubkey(GenerateNewKey(batch, internal)); - if (!batch.WritePool(index, CKeyPool(pubkey, internal))) { - throw std::runtime_error(std::string(__func__) + ": writing generated key failed"); - } - - if (internal) { - setInternalKeyPool.insert(index); - } else { - setExternalKeyPool.insert(index); - } - m_pool_key_to_index[pubkey.GetID()] = index; + AddKeypoolPubkeyWithDB(pubkey, internal, batch); } if (missingInternal + missingExternal > 0) { WalletLogPrintf("keypool added %d keys (%d internal), size=%u (%u internal)\n", missingInternal + missingExternal, missingInternal, setInternalKeyPool.size() + setExternalKeyPool.size() + set_pre_split_keypool.size(), setInternalKeyPool.size()); @@ -3345,6 +3448,29 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) return true; } +void CWallet::AddKeypoolPubkey(const CPubKey& pubkey, const bool internal) +{ + WalletBatch batch(*database); + AddKeypoolPubkeyWithDB(pubkey, internal, batch); + NotifyCanGetAddressesChanged(); +} + +void CWallet::AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch) +{ + LOCK(cs_wallet); + assert(m_max_keypool_index < std::numeric_limits<int64_t>::max()); // How in the hell did you use so many keys? + int64_t index = ++m_max_keypool_index; + if (!batch.WritePool(index, CKeyPool(pubkey, internal))) { + throw std::runtime_error(std::string(__func__) + ": writing imported pubkey failed"); + } + if (internal) { + setInternalKeyPool.insert(index); + } else { + setExternalKeyPool.insert(index); + } + m_pool_key_to_index[pubkey.GetID()] = index; +} + bool CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRequestedInternal) { nIndex = -1; @@ -3355,7 +3481,8 @@ bool CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRe if (!IsLocked()) TopUpKeyPool(); - bool fReturningInternal = IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT) && fRequestedInternal; + bool fReturningInternal = fRequestedInternal; + fReturningInternal &= (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) || IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); bool use_split_keypool = set_pre_split_keypool.empty(); std::set<int64_t>& setKeyPool = use_split_keypool ? (fReturningInternal ? setInternalKeyPool : setExternalKeyPool) : set_pre_split_keypool; @@ -3372,7 +3499,8 @@ bool CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool fRe if (!batch.ReadPool(nIndex, keypool)) { throw std::runtime_error(std::string(__func__) + ": read failed"); } - if (!HaveKey(keypool.vchPubKey.GetID())) { + CPubKey pk; + if (!GetPubKey(keypool.vchPubKey.GetID(), pk)) { throw std::runtime_error(std::string(__func__) + ": unknown key in key pool"); } // If the key was pre-split keypool, we don't care about what type it is @@ -3418,7 +3546,7 @@ void CWallet::ReturnKey(int64_t nIndex, bool fInternal, const CPubKey& pubkey) bool CWallet::GetKeyFromPool(CPubKey& result, bool internal) { - if (IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (!CanGetAddresses(internal)) { return false; } @@ -3426,7 +3554,7 @@ bool CWallet::GetKeyFromPool(CPubKey& result, bool internal) { LOCK(cs_wallet); int64_t nIndex; - if (!ReserveKeyFromKeyPool(nIndex, keypool, internal)) { + if (!ReserveKeyFromKeyPool(nIndex, keypool, internal) && !IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { if (IsLocked()) return false; WalletBatch batch(*database); result = GenerateNewKey(batch, internal); @@ -3512,7 +3640,7 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances(interfaces::Chain: std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() { - AssertLockHeld(cs_wallet); // mapWallet + AssertLockHeld(cs_wallet); std::set< std::set<CTxDestination> > groupings; std::set<CTxDestination> grouping; @@ -3619,6 +3747,10 @@ std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) co bool CReserveKey::GetReservedKey(CPubKey& pubkey, bool internal) { + if (!pwallet->CanGetAddresses(internal)) { + return false; + } + if (nIndex == -1) { CKeyPool keypool; @@ -3674,38 +3806,27 @@ void CWallet::MarkReserveKeysAsUsed(int64_t keypool_id) } } -void CWallet::GetScriptForMining(std::shared_ptr<CReserveScript> &script) -{ - std::shared_ptr<CReserveKey> rKey = std::make_shared<CReserveKey>(this); - CPubKey pubkey; - if (!rKey->GetReservedKey(pubkey)) - return; - - script = rKey; - script->reserveScript = CScript() << ToByteVector(pubkey) << OP_CHECKSIG; -} - void CWallet::LockCoin(const COutPoint& output) { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); setLockedCoins.insert(output); } void CWallet::UnlockCoin(const COutPoint& output) { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); setLockedCoins.erase(output); } void CWallet::UnlockAllCoins() { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); setLockedCoins.clear(); } bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); COutPoint outpt(hash, n); return (setLockedCoins.count(outpt) > 0); @@ -3713,7 +3834,7 @@ bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const { - AssertLockHeld(cs_wallet); // setLockedCoins + AssertLockHeld(cs_wallet); for (std::set<COutPoint>::iterator it = setLockedCoins.begin(); it != setLockedCoins.end(); it++) { COutPoint outpt = (*it); @@ -3723,8 +3844,8 @@ void CWallet::ListLockedCoins(std::vector<COutPoint>& vOutpts) const /** @} */ // end of Actions -void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<CTxDestination, int64_t> &mapKeyBirth) const { - AssertLockHeld(cs_wallet); // mapKeyMetadata +void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<CTxDestination, int64_t>& mapKeyBirth) const { + AssertLockHeld(cs_wallet); mapKeyBirth.clear(); // get birth times for keys with metadata @@ -3869,7 +3990,6 @@ bool CWallet::GetDestData(const CTxDestination &dest, const std::string &key, st std::vector<std::string> CWallet::GetDestValues(const std::string& prefix) const { - LOCK(cs_wallet); std::vector<std::string> values; for (const auto& address : mapAddressBook) { for (const auto& data : address.second.destdata) { @@ -3959,17 +4079,17 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, std::vector<CWalletTx> vWtx; if (gArgs.GetBoolArg("-zapwallettxes", false)) { - uiInterface.InitMessage(_("Zapping all transactions from wallet...")); + chain.initMessage(_("Zapping all transactions from wallet...")); std::unique_ptr<CWallet> tempWallet = MakeUnique<CWallet>(chain, location, WalletDatabase::Create(location.GetPath())); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DBErrors::LOAD_OK) { - InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); + chain.initError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); return nullptr; } } - uiInterface.InitMessage(_("Loading wallet...")); + chain.initMessage(_("Loading wallet...")); int64_t nStart = GetTimeMillis(); bool fFirstRun = true; @@ -3980,31 +4100,31 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (nLoadWalletRet != DBErrors::LOAD_OK) { if (nLoadWalletRet == DBErrors::CORRUPT) { - InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); + chain.initError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); return nullptr; } else if (nLoadWalletRet == DBErrors::NONCRITICAL_ERROR) { - InitWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data" - " or address book entries might be missing or incorrect."), + chain.initWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data" + " or address book entries might be missing or incorrect."), walletFile)); } else if (nLoadWalletRet == DBErrors::TOO_NEW) { - InitError(strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, _(PACKAGE_NAME))); + chain.initError(strprintf(_("Error loading %s: Wallet requires newer version of %s"), walletFile, _(PACKAGE_NAME))); return nullptr; } else if (nLoadWalletRet == DBErrors::NEED_REWRITE) { - InitError(strprintf(_("Wallet needed to be rewritten: restart %s to complete"), _(PACKAGE_NAME))); + chain.initError(strprintf(_("Wallet needed to be rewritten: restart %s to complete"), _(PACKAGE_NAME))); return nullptr; } else { - InitError(strprintf(_("Error loading %s"), walletFile)); + chain.initError(strprintf(_("Error loading %s"), walletFile)); return nullptr; } } - int prev_version = walletInstance->nWalletVersion; + int prev_version = walletInstance->GetVersion(); if (gArgs.GetBoolArg("-upgradewallet", fFirstRun)) { int nMaxVersion = gArgs.GetArg("-upgradewallet", 0); @@ -4018,7 +4138,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); if (nMaxVersion < walletInstance->GetVersion()) { - InitError(_("Cannot downgrade wallet")); + chain.initError(_("Cannot downgrade wallet")); return nullptr; } walletInstance->SetMaxVersion(nMaxVersion); @@ -4029,9 +4149,9 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, LOCK(walletInstance->cs_wallet); // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT - int max_version = walletInstance->nWalletVersion; - if (!walletInstance->CanSupportFeature(FEATURE_HD_SPLIT) && max_version >=FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { - InitError(_("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use -upgradewallet=169900 or -upgradewallet with no version specified.")); + int max_version = walletInstance->GetVersion(); + if (!walletInstance->CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { + chain.initError(_("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use -upgradewallet=169900 or -upgradewallet with no version specified.")); return nullptr; } @@ -4059,7 +4179,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // Regenerate the keypool if upgraded to HD if (hd_upgrade) { if (!walletInstance->TopUpKeyPool()) { - InitError(_("Unable to generate keys")); + chain.initError(_("Unable to generate keys")); return nullptr; } } @@ -4073,15 +4193,17 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if ((wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { //selective allow to set flags walletInstance->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + } else if (wallet_creation_flags & WALLET_FLAG_BLANK_WALLET) { + walletInstance->SetWalletFlag(WALLET_FLAG_BLANK_WALLET); } else { // generate a new seed CPubKey seed = walletInstance->GenerateNewSeed(); walletInstance->SetHDSeed(seed); - } + } // Otherwise, do not generate a new seed // Top up the keypool - if (!walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !walletInstance->TopUpKeyPool()) { - InitError(_("Unable to generate initial keys")); + if (walletInstance->CanGenerateKeys() && !walletInstance->TopUpKeyPool()) { + chain.initError(_("Unable to generate initial keys")); return nullptr; } @@ -4089,34 +4211,34 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->ChainStateFlushed(locked_chain->getTipLocator()); } else if (wallet_creation_flags & WALLET_FLAG_DISABLE_PRIVATE_KEYS) { // Make it impossible to disable private keys after creation - InitError(strprintf(_("Error loading %s: Private keys can only be disabled during creation"), walletFile)); + chain.initError(strprintf(_("Error loading %s: Private keys can only be disabled during creation"), walletFile)); return NULL; } else if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { LOCK(walletInstance->cs_KeyStore); if (!walletInstance->mapKeys.empty() || !walletInstance->mapCryptedKeys.empty()) { - InitWarning(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys"), walletFile)); + chain.initWarning(strprintf(_("Warning: Private keys detected in wallet {%s} with disabled private keys"), walletFile)); } } if (!gArgs.GetArg("-addresstype", "").empty() && !ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { - InitError(strprintf("Unknown address type '%s'", gArgs.GetArg("-addresstype", ""))); + chain.initError(strprintf("Unknown address type '%s'", gArgs.GetArg("-addresstype", ""))); return nullptr; } if (!gArgs.GetArg("-changetype", "").empty() && !ParseOutputType(gArgs.GetArg("-changetype", ""), walletInstance->m_default_change_type)) { - InitError(strprintf("Unknown change type '%s'", gArgs.GetArg("-changetype", ""))); + chain.initError(strprintf("Unknown change type '%s'", gArgs.GetArg("-changetype", ""))); return nullptr; } if (gArgs.IsArgSet("-mintxfee")) { CAmount n = 0; if (!ParseMoney(gArgs.GetArg("-mintxfee", ""), n) || 0 == n) { - InitError(AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", ""))); + chain.initError(AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", ""))); return nullptr; } if (n > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-mintxfee") + " " + - _("This is the minimum transaction fee you pay on every transaction.")); + chain.initWarning(AmountHighWarn("-mintxfee") + " " + + _("This is the minimum transaction fee you pay on every transaction.")); } walletInstance->m_min_fee = CFeeRate(n); } @@ -4125,12 +4247,12 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-fallbackfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-fallbackfee", ""), nFeePerK)) { - InitError(strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'"), gArgs.GetArg("-fallbackfee", ""))); + chain.initError(strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'"), gArgs.GetArg("-fallbackfee", ""))); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-fallbackfee") + " " + - _("This is the transaction fee you may pay when fee estimates are not available.")); + chain.initWarning(AmountHighWarn("-fallbackfee") + " " + + _("This is the transaction fee you may pay when fee estimates are not available.")); } walletInstance->m_fallback_fee = CFeeRate(nFeePerK); walletInstance->m_allow_fallback_fee = nFeePerK != 0; //disable fallback fee in case value was set to 0, enable if non-null value @@ -4138,28 +4260,28 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, if (gArgs.IsArgSet("-discardfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-discardfee", ""), nFeePerK)) { - InitError(strprintf(_("Invalid amount for -discardfee=<amount>: '%s'"), gArgs.GetArg("-discardfee", ""))); + chain.initError(strprintf(_("Invalid amount for -discardfee=<amount>: '%s'"), gArgs.GetArg("-discardfee", ""))); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-discardfee") + " " + - _("This is the transaction fee you may discard if change is smaller than dust at this level")); + chain.initWarning(AmountHighWarn("-discardfee") + " " + + _("This is the transaction fee you may discard if change is smaller than dust at this level")); } walletInstance->m_discard_rate = CFeeRate(nFeePerK); } if (gArgs.IsArgSet("-paytxfee")) { CAmount nFeePerK = 0; if (!ParseMoney(gArgs.GetArg("-paytxfee", ""), nFeePerK)) { - InitError(AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", ""))); + chain.initError(AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", ""))); return nullptr; } if (nFeePerK > HIGH_TX_FEE_PER_KB) { - InitWarning(AmountHighWarn("-paytxfee") + " " + - _("This is the transaction fee you will pay if you send a transaction.")); + chain.initWarning(AmountHighWarn("-paytxfee") + " " + + _("This is the transaction fee you will pay if you send a transaction.")); } walletInstance->m_pay_tx_fee = CFeeRate(nFeePerK, 1000); if (walletInstance->m_pay_tx_fee < ::minRelayTxFee) { - InitError(strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)"), + chain.initError(strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)"), gArgs.GetArg("-paytxfee", ""), ::minRelayTxFee.ToString())); return nullptr; } @@ -4200,20 +4322,19 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, //We can't rescan beyond non-pruned blocks, stop and throw an error //this might happen if a user uses an old wallet within a pruned node // or if he ran -disablewallet for a longer time, then decided to re-enable - if (fPruneMode) - { + if (chain.getPruneMode()) { int block_height = *tip_height; while (block_height > 0 && locked_chain->haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { --block_height; } if (rescan_height != block_height) { - InitError(_("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)")); + chain.initError(_("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)")); return nullptr; } } - uiInterface.InitMessage(_("Rescanning...")); + chain.initMessage(_("Rescanning...")); walletInstance->WalletLogPrintf("Rescanning last %i blocks (from block %i)...\n", *tip_height - rescan_height, rescan_height); // No need to read and scan block if block was created before @@ -4228,7 +4349,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, { WalletRescanReserver reserver(walletInstance.get()); if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), {} /* stop block */, reserver, true /* update */).status)) { - InitError(_("Failed to rescan the wallet during initialization")); + chain.initError(_("Failed to rescan the wallet during initialization")); return nullptr; } } @@ -4261,7 +4382,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - uiInterface.LoadWallet(walletInstance); + chain.loadWallet(interfaces::MakeWallet(walletInstance)); // Register with the validation interface. It's ok to do this after rescan since we're still holding cs_main. RegisterValidationInterface(walletInstance.get()); @@ -4344,17 +4465,14 @@ bool CMerkleTx::IsImmatureCoinBase(interfaces::Chain::Lock& locked_chain) const return GetBlocksToMaturity(locked_chain) > 0; } -bool CWalletTx::AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, const CAmount& nAbsurdFee, CValidationState& state) +bool CWalletTx::AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, CValidationState& state) { - LockAnnotation lock(::cs_main); // Temporary, for AcceptToMemoryPool below. Removed in upcoming commit. - // We must set fInMempool here - while it will be re-set to true by the // entered-mempool callback, if we did not there would be a race where a // user could call sendmoney in a loop and hit spurious out of funds errors // because we think that this newly generated transaction's change is // unavailable as we're not yet aware that it is in the mempool. - bool ret = ::AcceptToMemoryPool(mempool, state, tx, nullptr /* pfMissingInputs */, - nullptr /* plTxnReplaced */, false /* bypass_limits */, nAbsurdFee); + bool ret = locked_chain.submitToMemoryPool(tx, pwallet->chain().maxTxFee(), state); fInMempool |= ret; return ret; } @@ -4385,7 +4503,7 @@ std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outpu CInputCoin input_coin = output.GetInputCoin(); size_t ancestors, descendants; - mempool.GetTransactionAncestry(output.tx->GetHash(), ancestors, descendants); + chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); if (!single_coin && ExtractDestination(output.tx->tx->vout[output.i].scriptPubKey, dst)) { // Limit output groups to no more than 10 entries, to protect // against inadvertently creating a too-large transaction @@ -4416,18 +4534,21 @@ bool CWallet::GetKeyOrigin(const CKeyID& keyID, KeyOriginInfo& info) const meta = it->second; } } - if (!meta.hdKeypath.empty()) { - if (!ParseHDKeypath(meta.hdKeypath, info.path)) return false; - // Get the proper master key id - CKey key; - GetKey(meta.hd_seed_id, key); - CExtKey masterKey; - masterKey.SetSeed(key.begin(), key.size()); - // Compute identifier - CKeyID masterid = masterKey.key.GetPubKey().GetID(); - std::copy(masterid.begin(), masterid.begin() + 4, info.fingerprint); + if (meta.has_key_origin) { + std::copy(meta.key_origin.fingerprint, meta.key_origin.fingerprint + 4, info.fingerprint); + info.path = meta.key_origin.path; } else { // Single pubkeys get the master fingerprint of themselves std::copy(keyID.begin(), keyID.begin() + 4, info.fingerprint); } return true; } + +bool CWallet::AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info) +{ + LOCK(cs_wallet); + std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); + mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; + mapKeyMetadata[pubkey.GetID()].has_key_origin = true; + mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path); + return WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true); +} diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index cf3fa0aced..ea196c8799 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -66,6 +66,7 @@ bool RemoveWallet(const std::shared_ptr<CWallet>& wallet); bool HasWallets(); std::vector<std::shared_ptr<CWallet>> GetWallets(); std::shared_ptr<CWallet> GetWallet(const std::string& name); +std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const WalletLocation& location, std::string& error, std::string& warning); //! Default for -keypool static const unsigned int DEFAULT_KEYPOOL_SIZE = 1000; @@ -99,8 +100,6 @@ class CCoinControl; class COutput; class CReserveKey; class CScript; -class CTxMemPool; -class CBlockPolicyEstimator; class CWalletTx; struct FeeCalculation; enum class FeeEstimateMode; @@ -134,11 +133,26 @@ enum WalletFlags : uint64_t { // wallet flags in the upper section (> 1 << 31) will lead to not opening the wallet if flag is unknown // unknown wallet flags in the lower section <= (1 << 31) will be tolerated + // Indicates that the metadata has already been upgraded to contain key origins + WALLET_FLAG_KEY_ORIGIN_METADATA = (1ULL << 1), + // will enforce the rule that the wallet can't contain any private keys (only watch-only/pubkeys) WALLET_FLAG_DISABLE_PRIVATE_KEYS = (1ULL << 32), + + //! Flag set when a wallet contains no HD seed and no private keys, scripts, + //! addresses, and other watch only things, and is therefore "blank." + //! + //! The only function this flag serves is to distinguish a blank wallet from + //! a newly created wallet when the wallet database is loaded, to avoid + //! initialization that should only happen on first run. + //! + //! This flag is also a mandatory flag to prevent previous versions of + //! bitcoin from opening the wallet, thinking it was newly created, and + //! then improperly reinitializing it. + WALLET_FLAG_BLANK_WALLET = (1ULL << 33), }; -static constexpr uint64_t g_known_wallet_flags = WALLET_FLAG_DISABLE_PRIVATE_KEYS; +static constexpr uint64_t g_known_wallet_flags = WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET | WALLET_FLAG_KEY_ORIGIN_METADATA; /** A key pool entry */ class CKeyPool @@ -521,10 +535,10 @@ public: int64_t GetTxTime() const; // RelayWalletTransaction may only be called if fBroadcastTransactions! - bool RelayWalletTransaction(interfaces::Chain::Lock& locked_chain, CConnman* connman); + bool RelayWalletTransaction(interfaces::Chain::Lock& locked_chain); /** Pass this transaction to the mempool. Fails if absolute fee exceeds absurd fee. */ - bool AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, const CAmount& nAbsurdFee, CValidationState& state); + bool AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, CValidationState& state); // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation @@ -634,7 +648,7 @@ private: WalletBatch *encrypted_batch GUARDED_BY(cs_wallet) = nullptr; //! the current wallet version: clients below this version are not able to load the wallet - int nWalletVersion = FEATURE_BASE; + int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; //! the maximum wallet format version: memory-only variable that specifies to what version this wallet may be upgraded int nWalletMaxVersion GUARDED_BY(cs_wallet) = FEATURE_BASE; @@ -684,11 +698,11 @@ private: CHDChain hdChain; /* HD derive new child key (on internal or external chain) */ - void DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void DeriveNewChildKey(WalletBatch& batch, CKeyMetadata& metadata, CKey& secret, bool internal = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::set<int64_t> setInternalKeyPool; + std::set<int64_t> setInternalKeyPool GUARDED_BY(cs_wallet); std::set<int64_t> setExternalKeyPool GUARDED_BY(cs_wallet); - std::set<int64_t> set_pre_split_keypool; + std::set<int64_t> set_pre_split_keypool GUARDED_BY(cs_wallet); int64_t m_max_keypool_index GUARDED_BY(cs_wallet) = 0; std::map<CKeyID, int64_t> m_pool_key_to_index; std::atomic<uint64_t> m_wallet_flags{0}; @@ -763,6 +777,8 @@ public: // Map from Script ID to key metadata (for watch-only keys). std::map<CScriptID, CKeyMetadata> m_script_metadata GUARDED_BY(cs_wallet); + bool WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, bool overwrite); + typedef std::map<unsigned int, CMasterKey> MasterKeyMap; MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID = 0; @@ -788,7 +804,7 @@ public: int64_t nOrderPosNext GUARDED_BY(cs_wallet) = 0; uint64_t nAccountingEntryNumber = 0; - std::map<CTxDestination, CAddressBookData> mapAddressBook; + std::map<CTxDestination, CAddressBookData> mapAddressBook GUARDED_BY(cs_wallet); std::set<COutPoint> setLockedCoins GUARDED_BY(cs_wallet); @@ -853,6 +869,8 @@ public: //! Load metadata (used by LoadWallet) void LoadKeyMetadata(const CKeyID& keyID, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void LoadScriptMetadata(const CScriptID& script_id, const CKeyMetadata &metadata) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo + void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; nWalletMaxVersion = std::max(nWalletMaxVersion, nVersion); return true; } void UpdateTimeFirstKey(int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -865,15 +883,15 @@ public: bool LoadCScript(const CScript& redeemScript); //! Adds a destination data tuple to the store, and saves it to disk - bool AddDestData(const CTxDestination &dest, const std::string &key, const std::string &value); + bool AddDestData(const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Erases a destination data tuple in the store and on disk - bool EraseDestData(const CTxDestination &dest, const std::string &key); + bool EraseDestData(const CTxDestination& dest, const std::string& key) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a destination data tuple to the store, without saving it to disk - void LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value); + void LoadDestData(const CTxDestination& dest, const std::string& key, const std::string& value) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Look up a destination data tuple in the store, return true if found false otherwise - bool GetDestData(const CTxDestination &dest, const std::string &key, std::string *value) const; + bool GetDestData(const CTxDestination& dest, const std::string& key, std::string* value) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Get all destination values matching a prefix. - std::vector<std::string> GetDestValues(const std::string& prefix) const; + std::vector<std::string> GetDestValues(const std::string& prefix) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); //! Adds a watch-only address to the store, and saves it to disk. bool AddWatchOnly(const CScript& dest, int64_t nCreateTime) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -926,7 +944,7 @@ public: void ReacceptWalletTransactions(); void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override EXCLUSIVE_LOCKS_REQUIRED(cs_main); // ResendWalletTransactionsBefore may only be called if fBroadcastTransactions! - std::vector<uint256> ResendWalletTransactionsBefore(interfaces::Chain::Lock& locked_chain, int64_t nTime, CConnman* connman); + std::vector<uint256> ResendWalletTransactionsBefore(interfaces::Chain::Lock& locked_chain, int64_t nTime); CAmount GetBalance(const isminefilter& filter=ISMINE_SPENDABLE, const int min_depth=0) const; CAmount GetUnconfirmedBalance() const; CAmount GetImmatureBalance() const; @@ -951,7 +969,7 @@ public: */ bool CreateTransaction(interfaces::Chain::Lock& locked_chain, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CReserveKey& reservekey, CAmount& nFeeRet, int& nChangePosInOut, std::string& strFailReason, const CCoinControl& coin_control, bool sign = true); - bool CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CConnman* connman, CValidationState& state); + bool CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm, CReserveKey& reservekey, CValidationState& state); bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, bool use_max_sig = false) const { @@ -981,6 +999,8 @@ public: bool NewKeyPool(); size_t KeypoolCountExternalKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool TopUpKeyPool(unsigned int kpSize = 0); + void AddKeypoolPubkey(const CPubKey& pubkey, const bool internal); + void AddKeypoolPubkeyWithDB(const CPubKey& pubkey, const bool internal, WalletBatch& batch); /** * Reserves a key from the keypool and sets nIndex to its index @@ -1041,13 +1061,11 @@ public: bool DelAddressBook(const CTxDestination& address); - const std::string& GetLabelName(const CScript& scriptPubKey) const; - - void GetScriptForMining(std::shared_ptr<CReserveScript> &script); + const std::string& GetLabelName(const CScript& scriptPubKey) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); unsigned int GetKeyPoolSize() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { - AssertLockHeld(cs_wallet); // set{Ex,In}ternalKeyPool + AssertLockHeld(cs_wallet); return setInternalKeyPool.size() + setExternalKeyPool.size(); } @@ -1132,6 +1150,12 @@ public: /* Returns true if HD is enabled */ bool IsHDEnabled() const; + /* Returns true if the wallet can generate new keys */ + bool CanGenerateKeys(); + + /* Returns true if the wallet can give out new addresses. This means it has keys in the keypool or can generate new keys */ + bool CanGetAddresses(bool internal = false); + /* Generates a new HD seed (will not be activated) */ CPubKey GenerateNewSeed(); @@ -1169,6 +1193,9 @@ public: /** set a single wallet flag */ void SetWalletFlag(uint64_t flags); + /** Unsets a single wallet flag */ + void UnsetWalletFlag(uint64_t flag); + /** check if a certain wallet flag is set */ bool IsWalletFlagSet(uint64_t flag); @@ -1190,6 +1217,9 @@ public: /** Implement lookup of key origin information through wallet key metadata. */ bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override; + + /** Add a KeyOriginInfo to the wallet */ + bool AddKeyOrigin(const CPubKey& pubkey, const KeyOriginInfo& info); }; /** A key allocated from the key pool. */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 6e037808e3..2783f83fd6 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -57,9 +57,14 @@ bool WalletBatch::EraseTx(uint256 hash) return EraseIC(std::make_pair(std::string("tx"), hash)); } +bool WalletBatch::WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, const bool overwrite) +{ + return WriteIC(std::make_pair(std::string("keymeta"), pubkey), meta, overwrite); +} + bool WalletBatch::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, const CKeyMetadata& keyMeta) { - if (!WriteIC(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta, false)) { + if (!WriteKeyMetadata(keyMeta, vchPubKey, false)) { return false; } @@ -76,7 +81,7 @@ bool WalletBatch::WriteCryptedKey(const CPubKey& vchPubKey, const std::vector<unsigned char>& vchCryptedSecret, const CKeyMetadata &keyMeta) { - if (!WriteIC(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta)) { + if (!WriteKeyMetadata(keyMeta, vchPubKey, true)) { return false; } @@ -529,6 +534,14 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (wss.fAnyUnordered) result = pwallet->ReorderTransactions(); + // Upgrade all of the wallet keymetadata to have the hd master key id + // This operation is not atomic, but if it fails, updated entries are still backwards compatible with older software + try { + pwallet->UpgradeKeyMetadata(); + } catch (...) { + result = DBErrors::CORRUPT; + } + return result; } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 5584407a56..0532a55ff5 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -8,6 +8,7 @@ #include <amount.h> #include <primitives/transaction.h> +#include <script/sign.h> #include <wallet/db.h> #include <key.h> @@ -93,11 +94,14 @@ class CKeyMetadata public: static const int VERSION_BASIC=1; static const int VERSION_WITH_HDDATA=10; - static const int CURRENT_VERSION=VERSION_WITH_HDDATA; + static const int VERSION_WITH_KEY_ORIGIN = 12; + static const int CURRENT_VERSION=VERSION_WITH_KEY_ORIGIN; int nVersion; int64_t nCreateTime; // 0 means unknown - std::string hdKeypath; //optional HD/bip32 keypath + std::string hdKeypath; //optional HD/bip32 keypath. Still used to determine whether a key is a seed. Also kept for backwards compatibility CKeyID hd_seed_id; //id of the HD seed used to derive this key + KeyOriginInfo key_origin; // Key origin info with path and fingerprint + bool has_key_origin = false; //< Whether the key_origin is useful CKeyMetadata() { @@ -120,6 +124,11 @@ public: READWRITE(hdKeypath); READWRITE(hd_seed_id); } + if (this->nVersion >= VERSION_WITH_KEY_ORIGIN) + { + READWRITE(key_origin); + READWRITE(has_key_origin); + } } void SetNull() @@ -128,6 +137,8 @@ public: nCreateTime = 0; hdKeypath.clear(); hd_seed_id.SetNull(); + key_origin.clear(); + has_key_origin = false; } }; @@ -177,6 +188,7 @@ public: bool WriteTx(const CWalletTx& wtx); bool EraseTx(uint256 hash); + bool WriteKeyMetadata(const CKeyMetadata& meta, const CPubKey& pubkey, const bool overwrite); bool WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, const CKeyMetadata &keyMeta); bool WriteCryptedKey(const CPubKey& vchPubKey, const std::vector<unsigned char>& vchCryptedSecret, const CKeyMetadata &keyMeta); bool WriteMasterKey(unsigned int nID, const CMasterKey& kMasterKey); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 30b0c48eef..797f051189 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -17,7 +17,7 @@ namespace WalletTool { static void WalletToolReleaseWallet(CWallet* wallet) { wallet->WalletLogPrintf("Releasing wallet\n"); - wallet->Flush(); + wallet->Flush(true); delete wallet; } @@ -94,7 +94,6 @@ static std::shared_ptr<CWallet> LoadWallet(const std::string& name, const fs::pa static void WalletShowInfo(CWallet* wallet_instance) { - // lock required because of some AssertLockHeld() LOCK(wallet_instance->cs_wallet); fprintf(stdout, "Wallet info\n===========\n"); @@ -113,7 +112,7 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) std::shared_ptr<CWallet> wallet_instance = CreateWallet(name, path); if (wallet_instance) { WalletShowInfo(wallet_instance.get()); - wallet_instance->Flush(); + wallet_instance->Flush(true); } } else if (command == "info") { if (!fs::exists(path)) { @@ -128,7 +127,7 @@ bool ExecuteWalletToolFunc(const std::string& command, const std::string& name) std::shared_ptr<CWallet> wallet_instance = LoadWallet(name, path); if (!wallet_instance) return false; WalletShowInfo(wallet_instance.get()); - wallet_instance->Flush(); + wallet_instance->Flush(true); } else { fprintf(stderr, "Invalid command: %s\n", command.c_str()); return false; diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 6db4c63acb..d779251d56 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -35,7 +35,7 @@ static bool IsBerkeleyBtree(const fs::path& path) boost::system::error_code ec; if (fs::file_size(path, ec) < 4096) return false; - fs::ifstream file(path.string(), std::ios::binary); + fsbridge::ifstream file(path, std::ios::binary); if (!file.is_open()) return false; file.seekg(12, std::ios::beg); // Magic bytes start at offset 12 diff --git a/test/config.ini.in b/test/config.ini.in index 28abee2a3d..6b7ef70659 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -16,4 +16,5 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @ENABLE_WALLET_TRUE@ENABLE_WALLET=true @BUILD_BITCOIN_CLI_TRUE@ENABLE_CLI=true @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true +@ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true diff --git a/test/functional/README.md b/test/functional/README.md index 74f454b86c..5e3009e6af 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -43,6 +43,7 @@ don't have test cases for. - `mining` for tests for mining features, eg `mining_prioritisetransaction.py` - `p2p` for tests that explicitly test the p2p interface, eg `p2p_disconnect_ban.py` - `rpc` for tests for individual RPC methods or features, eg `rpc_listtransactions.py` + - `tool` for tests for tools, eg `tool_wallet.py` - `wallet` for tests for wallet features, eg `wallet_keypool.py` - use an underscore to separate words - exception: for tests for specific RPCs or command line options which don't include underscores, name the test after the exact RPC or argument name, eg `rpc_decodescript.py`, not `rpc_decode_script.py` diff --git a/test/functional/example_test.py b/test/functional/example_test.py index f367e4fca8..0e70ebba6d 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """An example functional test diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index 12a4ce9aff..e7e4f84ad9 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for skipping signature validation on old blocks. @@ -180,7 +180,7 @@ class AssumeValidTest(BitcoinTestFramework): for i in range(2202): p2p1.send_message(msg_block(self.blocks[i])) # Syncing 2200 blocks can take a while on slow systems. Give it plenty of time to sync. - p2p1.sync_with_ping(150) + p2p1.sync_with_ping(200) assert_equal(self.nodes[1].getblock(self.nodes[1].getbestblockhash())['height'], 2202) # Send blocks to node2. Block 102 will be rejected. diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 8466f851ca..21c832c1a9 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP68 implementation.""" @@ -10,7 +10,7 @@ from test_framework.blocktools import create_block, create_coinbase, add_witness from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, ToHex from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, bytes_to_hex_str, get_bip9_status, satoshi_round, sync_blocks +from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, get_bip9_status, satoshi_round, sync_blocks SEQUENCE_LOCKTIME_DISABLE_FLAG = (1<<31) SEQUENCE_LOCKTIME_TYPE_FLAG = (1<<22) # this means use time (0 means height) @@ -63,7 +63,7 @@ class BIP68Test(BitcoinTestFramework): self.nodes[0].sendtoaddress(new_addr, 2) # send 2 BTC utxos = self.nodes[0].listunspent(0, 0) - assert(len(utxos) > 0) + assert len(utxos) > 0 utxo = utxos[0] @@ -253,7 +253,7 @@ class BIP68Test(BitcoinTestFramework): self.nodes[0].generate(1) cur_time += 600 - assert(tx2.hash in self.nodes[0].getrawmempool()) + assert tx2.hash in self.nodes[0].getrawmempool() test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=True) test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=False) @@ -264,23 +264,23 @@ class BIP68Test(BitcoinTestFramework): # Advance the time on the node so that we can test timelocks self.nodes[0].setmocktime(cur_time+600) self.nodes[0].generate(1) - assert(tx2.hash not in self.nodes[0].getrawmempool()) + assert tx2.hash not in self.nodes[0].getrawmempool() # Now that tx2 is not in the mempool, a sequence locked spend should # succeed tx3 = test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=False) - assert(tx3.hash in self.nodes[0].getrawmempool()) + assert tx3.hash in self.nodes[0].getrawmempool() self.nodes[0].generate(1) - assert(tx3.hash not in self.nodes[0].getrawmempool()) + assert tx3.hash not in self.nodes[0].getrawmempool() # One more test, this time using height locks tx4 = test_nonzero_locks(tx3, self.nodes[0], self.relayfee, use_height_lock=True) - assert(tx4.hash in self.nodes[0].getrawmempool()) + assert tx4.hash in self.nodes[0].getrawmempool() # Now try combining confirmed and unconfirmed inputs tx5 = test_nonzero_locks(tx4, self.nodes[0], self.relayfee, use_height_lock=True) - assert(tx5.hash not in self.nodes[0].getrawmempool()) + assert tx5.hash not in self.nodes[0].getrawmempool() utxos = self.nodes[0].listunspent() tx5.vin.append(CTxIn(COutPoint(int(utxos[0]["txid"], 16), utxos[0]["vout"]), nSequence=1)) @@ -299,8 +299,8 @@ class BIP68Test(BitcoinTestFramework): # If we invalidate the tip, tx3 should get added to the mempool, causing # tx4 to be removed (fails sequence-lock). self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) - assert(tx4.hash not in self.nodes[0].getrawmempool()) - assert(tx3.hash in self.nodes[0].getrawmempool()) + assert tx4.hash not in self.nodes[0].getrawmempool() + assert tx3.hash in self.nodes[0].getrawmempool() # Now mine 2 empty blocks to reorg out the current tip (labeled tip-1 in # diagram above). @@ -319,8 +319,8 @@ class BIP68Test(BitcoinTestFramework): cur_time += 1 mempool = self.nodes[0].getrawmempool() - assert(tx3.hash not in mempool) - assert(tx2.hash in mempool) + assert tx3.hash not in mempool + assert tx2.hash in mempool # Reset the chain and get rid of the mocktimed-blocks self.nodes[0].setmocktime(0) @@ -332,7 +332,7 @@ class BIP68Test(BitcoinTestFramework): # being run, then it's possible the test has activated the soft fork, and # this test should be moved to run earlier, or deleted. def test_bip68_not_consensus(self): - assert(get_bip9_status(self.nodes[0], 'csv')['status'] != 'active') + assert get_bip9_status(self.nodes[0], 'csv')['status'] != 'active' txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 2) tx1 = FromHex(CTransaction(), self.nodes[0].getrawtransaction(txid)) @@ -372,7 +372,7 @@ class BIP68Test(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) assert_equal(self.nodes[0].getbestblockhash(), block.hash) def activateCSV(self): diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 5253ff7aaa..faf7f20257 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -773,7 +773,7 @@ class FullBlockTest(BitcoinTestFramework): self.move_tip(57) b58 = self.next_block(58, spend=out[17]) tx = CTransaction() - assert(len(out[17].vout) < 42) + assert len(out[17].vout) < 42 tx.vin.append(CTxIn(COutPoint(out[17].sha256, 42), CScript([OP_TRUE]), 0xffffffff)) tx.vout.append(CTxOut(0, b"")) tx.calc_sha256() @@ -824,7 +824,7 @@ class FullBlockTest(BitcoinTestFramework): tx.nLockTime = 0xffffffff # this locktime is non-final tx.vin.append(CTxIn(COutPoint(out[18].sha256, 0))) # don't set nSequence tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) - assert(tx.vin[0].nSequence < 0xffffffff) + assert tx.vin[0].nSequence < 0xffffffff tx.calc_sha256() b62 = self.update_block(62, [tx]) self.sync_blocks([b62], success=False, reject_reason='bad-txns-nonfinal') @@ -1143,8 +1143,8 @@ class FullBlockTest(BitcoinTestFramework): # now check that tx78 and tx79 have been put back into the peer's mempool mempool = self.nodes[0].getrawmempool() assert_equal(len(mempool), 2) - assert(tx78.hash in mempool) - assert(tx79.hash in mempool) + assert tx78.hash in mempool + assert tx79.hash in mempool # Test invalid opcodes in dead execution paths. # diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 302a5ec1cb..070242c1df 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP65 (CHECKLOCKTIMEVERIFY). @@ -15,7 +15,6 @@ from test_framework.script import CScript, OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, O from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, hex_str_to_bytes, ) @@ -60,6 +59,7 @@ class BIP65Test(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [['-whitelist=127.0.0.1', '-par=1']] # Use only one script thread to get the exact reject reason for testing self.setup_clean_chain = True + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -113,7 +113,7 @@ class BIP65Test(BitcoinTestFramework): # rejected from the mempool for exactly that reason. assert_equal( [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Negative locktime)'}], - self.nodes[0].testmempoolaccept(rawtxs=[bytes_to_hex_str(spendtx.serialize())], allowhighfees=True) + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], allowhighfees=True) ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 4b3f6603a2..13481b0478 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various command line arguments and configuration file parameters.""" @@ -41,13 +41,21 @@ class ConfArgsTest(BitcoinTestFramework): conf.write('server=1\nrpcuser=someuser\n[main]\nrpcpassword=some#pass') self.nodes[0].assert_start_raises_init_error(expected_msg='Error reading configuration file: parse error on line 4, using # in rpcpassword can be ambiguous and should be avoided') + inc_conf_file2_path = os.path.join(self.nodes[0].datadir, 'include2.conf') + with open(os.path.join(self.nodes[0].datadir, 'bitcoin.conf'), 'a', encoding='utf-8') as conf: + conf.write('includeconf={}\n'.format(inc_conf_file2_path)) + with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: - conf.write('testnot.datadir=1\n[testnet]\n') + conf.write('testnot.datadir=1\n') + with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: + conf.write('[testnet]\n') self.restart_node(0) - self.nodes[0].stop_node(expected_stderr='Warning: Section [testnet] is not recognized.' + os.linesep + 'Warning: Section [testnot] is not recognized.') + self.nodes[0].stop_node(expected_stderr='Warning: ' + inc_conf_file_path + ':1 Section [testnot] is not recognized.' + os.linesep + 'Warning: ' + inc_conf_file2_path + ':1 Section [testnet] is not recognized.') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('') # clear + with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: + conf.write('') # clear def run_test(self): self.stop_node(0) diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index df79c4312c..ecc68217bb 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test activation of the first version bits soft fork. @@ -106,7 +106,7 @@ def send_generic_input_tx(node, coinbases, address): def create_bip68txs(node, bip68inputs, txversion, address, locktime_delta=0): """Returns a list of bip68 transactions with different bits set.""" txs = [] - assert(len(bip68inputs) >= 16) + assert len(bip68inputs) >= 16 for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = create_transaction(node, bip68inputs[i], address, amount=Decimal("49.98")) @@ -121,7 +121,7 @@ def create_bip68txs(node, bip68inputs, txversion, address, locktime_delta=0): def create_bip112txs(node, bip112inputs, varyOP_CSV, txversion, address, locktime_delta=0): """Returns a list of bip68 transactions with different bits set.""" txs = [] - assert(len(bip112inputs) >= 16) + assert len(bip112inputs) >= 16 for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = create_transaction(node, bip112inputs[i], address, amount=Decimal("49.98")) diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 8b06cc7372..62062926a6 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test recovery from a crash during chainstate writing. @@ -28,18 +28,12 @@ import errno import http.client import random -import sys import time from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, ToHex from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, create_confirmed_utxos, hex_str_to_bytes -HTTP_DISCONNECT_ERRORS = [http.client.CannotSendRequest] -try: - HTTP_DISCONNECT_ERRORS.append(http.client.RemoteDisconnected) -except AttributeError: - pass class ChainstateWriteCrashTest(BitcoinTestFramework): def set_test_params(self): @@ -110,14 +104,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): try: self.nodes[node_index].submitblock(block) return True - except http.client.BadStatusLine as e: - # Prior to 3.5 BadStatusLine('') was raised for a remote disconnect error. - if sys.version_info[0] == 3 and sys.version_info[1] < 5 and e.line == "''": - self.log.debug("node %d submitblock raised exception: %s", node_index, e) - return False - else: - raise - except tuple(HTTP_DISCONNECT_ERRORS) as e: + except (http.client.CannotSendRequest, http.client.RemoteDisconnected) as e: self.log.debug("node %d submitblock raised exception: %s", node_index, e) return False except OSError as e: diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 9cbc1b39bd..4ddfd80b07 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP66 (DER SIG). @@ -14,7 +14,6 @@ from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, wait_until, ) @@ -47,6 +46,7 @@ class BIP66Test(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [['-whitelist=127.0.0.1', '-par=1', '-enablebip61']] # Use only one script thread to get the exact reject reason for testing self.setup_clean_chain = True + self.rpc_timeout = 120 def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -102,7 +102,7 @@ class BIP66Test(BitcoinTestFramework): # rejected from the mempool for exactly that reason. assert_equal( [{'txid': spendtx.hash, 'allowed': False, 'reject-reason': '64: non-mandatory-script-verify-flag (Non-canonical DER signature)'}], - self.nodes[0].testmempoolaccept(rawtxs=[bytes_to_hex_str(spendtx.serialize())], allowhighfees=True) + self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], allowhighfees=True) ) # Now we verify that a block with this transaction is also invalid. diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index d8083b2840..13cf951550 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the -alertnotify, -blocknotify and -walletnotify options.""" @@ -66,23 +66,7 @@ class NotificationsTest(BitcoinTestFramework): txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) assert_equal(sorted(txids_rpc), sorted(os.listdir(self.walletnotify_dir))) - # Mine another 41 up-version blocks. -alertnotify should trigger on the 51st. - self.log.info("test -alertnotify") - self.nodes[1].generatetoaddress(41, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() - - # Give bitcoind 10 seconds to write the alert notification - wait_until(lambda: len(os.listdir(self.alertnotify_dir)), timeout=10) - - for notify_file in os.listdir(self.alertnotify_dir): - os.remove(os.path.join(self.alertnotify_dir, notify_file)) - - # Mine more up-version blocks, should not get more alerts: - self.nodes[1].generatetoaddress(2, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() - - self.log.info("-alertnotify should not continue notifying for more unknown version blocks") - assert_equal(len(os.listdir(self.alertnotify_dir)), 0) + # TODO: add test for `-alertnotify` large fork notifications if __name__ == '__main__': NotificationsTest().main() diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index eb76089d9c..e5d2f8dc46 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test NULLDUMMY softfork. @@ -18,7 +18,7 @@ from test_framework.blocktools import create_coinbase, create_block, create_tran from test_framework.messages import CTransaction from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str +from test_framework.util import assert_equal, assert_raises_rpc_error NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero) (code 64)" @@ -27,7 +27,7 @@ def trueDummy(tx): newscript = [] for i in scriptSig: if (len(newscript) == 0): - assert(len(i) == 0) + assert len(i) == 0 newscript.append(b'\x51') else: newscript.append(i) @@ -64,17 +64,17 @@ class NULLDUMMYTest(BitcoinTestFramework): self.log.info("Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [430]") test1txs = [create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, amount=49)] - txid1 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[0].serialize_with_witness()), True) + txid1 = self.nodes[0].sendrawtransaction(test1txs[0].serialize_with_witness().hex(), True) test1txs.append(create_transaction(self.nodes[0], txid1, self.ms_address, amount=48)) - txid2 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[1].serialize_with_witness()), True) + txid2 = self.nodes[0].sendrawtransaction(test1txs[1].serialize_with_witness().hex(), True) test1txs.append(create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, amount=49)) - txid3 = self.nodes[0].sendrawtransaction(bytes_to_hex_str(test1txs[2].serialize_with_witness()), True) + txid3 = self.nodes[0].sendrawtransaction(test1txs[2].serialize_with_witness().hex(), True) self.block_submit(self.nodes[0], test1txs, False, True) self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") test2tx = create_transaction(self.nodes[0], txid2, self.ms_address, amount=47) trueDummy(test2tx) - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test2tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test2tx.serialize_with_witness().hex(), True) self.log.info("Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [431]") self.block_submit(self.nodes[0], [test2tx], False, True) @@ -83,19 +83,19 @@ class NULLDUMMYTest(BitcoinTestFramework): test4tx = create_transaction(self.nodes[0], test2tx.hash, self.address, amount=46) test6txs = [CTransaction(test4tx)] trueDummy(test4tx) - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test4tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test4tx.serialize_with_witness().hex(), True) self.block_submit(self.nodes[0], [test4tx]) self.log.info("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation") test5tx = create_transaction(self.nodes[0], txid3, self.wit_address, amount=48) test6txs.append(CTransaction(test5tx)) test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01' - assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, bytes_to_hex_str(test5tx.serialize_with_witness()), True) + assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test5tx.serialize_with_witness().hex(), True) self.block_submit(self.nodes[0], [test5tx], True) self.log.info("Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [432]") for i in test6txs: - self.nodes[0].sendrawtransaction(bytes_to_hex_str(i.serialize_with_witness()), True) + self.nodes[0].sendrawtransaction(i.serialize_with_witness().hex(), True) self.block_submit(self.nodes[0], test6txs, True, True) def block_submit(self, node, txs, witness=False, accept=False): @@ -108,7 +108,7 @@ class NULLDUMMYTest(BitcoinTestFramework): witness and add_witness_commitment(block) block.rehash() block.solve() - node.submitblock(bytes_to_hex_str(block.serialize(True))) + node.submitblock(block.serialize(True).hex()) if (accept): assert_equal(node.getbestblockhash(), block.hash) self.tip = block.sha256 diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index 31d2ee8e13..be323d355e 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind with different proxy configuration. @@ -44,6 +44,7 @@ RANGE_BEGIN = PORT_MIN + 2 * PORT_RANGE # Start after p2p and rpc ports class ProxyTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 + self.setup_clean_chain = True def setup_nodes(self): self.have_ipv6 = test_ipv6_local() @@ -94,7 +95,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing IPv4 connection through node node.addnode("15.61.23.23:1234", "onetry") cmd = proxies[0].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"15.61.23.23") @@ -108,7 +109,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing IPv6 connection through node node.addnode("[1233:3432:2434:2343:3234:2345:6546:4534]:5443", "onetry") cmd = proxies[1].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"1233:3432:2434:2343:3234:2345:6546:4534") @@ -122,7 +123,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing onion connection through node node.addnode("bitcoinostk4e4re.onion:8333", "onetry") cmd = proxies[2].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"bitcoinostk4e4re.onion") assert_equal(cmd.port, 8333) @@ -134,7 +135,7 @@ class ProxyTest(BitcoinTestFramework): # Test: outgoing DNS name connection through node node.addnode("node.noumenon:8333", "onetry") cmd = proxies[3].queue.get() - assert(isinstance(cmd, Socks5Command)) + assert isinstance(cmd, Socks5Command) assert_equal(cmd.atyp, AddressType.DOMAINNAME) assert_equal(cmd.addr, b"node.noumenon") assert_equal(cmd.port, 8333) @@ -198,4 +199,3 @@ class ProxyTest(BitcoinTestFramework): if __name__ == '__main__': ProxyTest().main() - diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 3e1ba88f0a..12da2655ee 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the pruning code. @@ -151,7 +151,7 @@ class PruneTest(BitcoinTestFramework): self.nodes[1].invalidateblock(curhash) curhash = self.nodes[1].getblockhash(invalidheight - 1) - assert(self.nodes[1].getblockcount() == invalidheight - 1) + assert self.nodes[1].getblockcount() == invalidheight - 1 self.log.info("New best height: %d" % self.nodes[1].getblockcount()) # Reboot node1 to clear those giant tx's from mempool @@ -219,17 +219,17 @@ class PruneTest(BitcoinTestFramework): blocks_to_mine = first_reorg_height + 1 - self.mainchainheight self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine) self.nodes[0].invalidateblock(curchainhash) - assert(self.nodes[0].getblockcount() == self.mainchainheight) - assert(self.nodes[0].getbestblockhash() == self.mainchainhash2) + assert self.nodes[0].getblockcount() == self.mainchainheight + assert self.nodes[0].getbestblockhash() == self.mainchainhash2 goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] goalbestheight = first_reorg_height + 1 self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") # Wait for Node 2 to reorg to proper height wait_until(lambda: self.nodes[2].getblockcount() >= goalbestheight, timeout=900) - assert(self.nodes[2].getbestblockhash() == goalbesthash) + assert self.nodes[2].getbestblockhash() == goalbesthash # Verify we can now have the data for a block previously pruned - assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight) + assert self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight def manual_test(self, node_number, use_timestamp): # at this point, node has 995 blocks and has not yet run in prune mode diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index ff7c2b23bf..a9c7226e46 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" @@ -9,12 +9,12 @@ from decimal import Decimal from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut from test_framework.script import CScript, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, satoshi_round +from test_framework.util import assert_equal, assert_raises_rpc_error, satoshi_round MAX_REPLACEMENT_LIMIT = 100 def txToHex(tx): - return bytes_to_hex_str(tx.serialize()) + return tx.serialize().hex() def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): """Create a txout with a given amount and scriptPubKey @@ -56,7 +56,7 @@ def make_utxo(node, amount, confirmed=True, scriptPubKey=CScript([1])): new_size = len(node.getrawmempool()) # Error out if we have something stuck in the mempool, as this # would likely be a bug. - assert(new_size < mempool_size) + assert new_size < mempool_size mempool_size = new_size return COutPoint(int(txid, 16), 0) @@ -163,8 +163,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): mempool = self.nodes[0].getrawmempool() - assert (tx1a_txid not in mempool) - assert (tx1b_txid in mempool) + assert tx1a_txid not in mempool + assert tx1b_txid in mempool assert_equal(tx1b_hex, self.nodes[0].getrawtransaction(tx1b_txid)) @@ -211,7 +211,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): mempool = self.nodes[0].getrawmempool() for doublespent_txid in chain_txids: - assert(doublespent_txid not in mempool) + assert doublespent_txid not in mempool def test_doublespend_tree(self): """Doublespend of a big tree of transactions""" @@ -236,7 +236,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx.vout = vout tx_hex = txToHex(tx) - assert(len(tx.serialize()) < 100000) + assert len(tx.serialize()) < 100000 txid = self.nodes[0].sendrawtransaction(tx_hex, True) yield tx _total_txs[0] += 1 @@ -274,7 +274,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): for tx in tree_txs: tx.rehash() - assert (tx.hash not in mempool) + assert tx.hash not in mempool # Try again, but with more total transactions than the "max txs # double-spent at once" anti-DoS limit. @@ -529,7 +529,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): # Now tx1b should be able to replace tx1a tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, True) - assert(tx1b_txid in self.nodes[0].getrawmempool()) + assert tx1b_txid in self.nodes[0].getrawmempool() # 2. Check that absolute fee checks use modified fee. tx1_outpoint = make_utxo(self.nodes[0], int(1.1*COIN)) @@ -556,7 +556,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): # tx2b should now be accepted tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, True) - assert(tx2b_txid in self.nodes[0].getrawmempool()) + assert tx2b_txid in self.nodes[0].getrawmempool() def test_rpc(self): us0 = self.nodes[0].listunspent()[0] diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 1efc50e71f..0128af8b65 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the SegWit changeover logic.""" @@ -18,7 +18,7 @@ from test_framework.blocktools import witness_script, send_to_witness from test_framework.messages import COIN, COutPoint, CTransaction, CTxIn, CTxOut, FromHex, sha256, ToHex from test_framework.script import CScript, OP_HASH160, OP_CHECKSIG, OP_0, hash160, OP_EQUAL, OP_DUP, OP_EQUALVERIFY, OP_1, OP_2, OP_CHECKMULTISIG, OP_TRUE, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes, hex_str_to_bytes, sync_blocks, try_rpc +from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes, hex_str_to_bytes, sync_blocks, try_rpc NODE_0 = 0 NODE_2 = 2 @@ -38,31 +38,29 @@ def find_spendable_utxo(node, min_value): raise AssertionError("Unspent output equal or higher than %s not found" % min_value) +txs_mined = {} # txindex from txid to blockhash + class SegWitTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. - # TODO: remove -txindex. Currently required for getrawtransaction call. self.extra_args = [ [ "-rpcserialversion=0", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", - "-txindex" ], [ "-blockversion=4", "-rpcserialversion=1", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", - "-txindex" ], [ "-blockversion=536870915", "-vbparams=segwit:0:999999999999", "-addresstype=legacy", - "-txindex" ], ] @@ -95,17 +93,17 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify sigops are counted in GBT with pre-BIP141 rules before the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] == 1000000) - assert('weightlimit' not in tmpl) - assert(tmpl['sigoplimit'] == 20000) - assert(tmpl['transactions'][0]['hash'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 2) + assert tmpl['sizelimit'] == 1000000 + assert 'weightlimit' not in tmpl + assert tmpl['sigoplimit'] == 20000 + assert tmpl['transactions'][0]['hash'] == txid + assert tmpl['transactions'][0]['sigops'] == 2 tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] == 1000000) - assert('weightlimit' not in tmpl) - assert(tmpl['sigoplimit'] == 20000) - assert(tmpl['transactions'][0]['hash'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 2) + assert tmpl['sizelimit'] == 1000000 + assert 'weightlimit' not in tmpl + assert tmpl['sigoplimit'] == 20000 + assert tmpl['transactions'][0]['hash'] == txid + assert tmpl['transactions'][0]['sigops'] == 2 self.nodes[0].generate(1) # block 162 balance_presetup = self.nodes[0].getbalance() @@ -157,10 +155,10 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) - block = self.nodes[2].generate(1) # block 432 (first block with new rules; 432 = 144 * 3) + blockhash = self.nodes[2].generate(1)[0] # block 432 (first block with new rules; 432 = 144 * 3) sync_blocks(self.nodes) assert_equal(len(self.nodes[2].getrawmempool()), 0) - segwit_tx_list = self.nodes[2].getblock(block[0])["tx"] + segwit_tx_list = self.nodes[2].getblock(blockhash)["tx"] assert_equal(len(segwit_tx_list), 5) self.log.info("Verify default node can't accept txs with missing witness") @@ -174,15 +172,16 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False, witness_script(True, self.pubkey[0])) self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") - assert(self.nodes[2].getblock(block[0], False) != self.nodes[0].getblock(block[0], False)) - assert(self.nodes[1].getblock(block[0], False) == self.nodes[2].getblock(block[0], False)) - for i in range(len(segwit_tx_list)): - tx = FromHex(CTransaction(), self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) - assert(self.nodes[2].getrawtransaction(segwit_tx_list[i]) != self.nodes[0].getrawtransaction(segwit_tx_list[i])) - assert(self.nodes[1].getrawtransaction(segwit_tx_list[i], 0) == self.nodes[2].getrawtransaction(segwit_tx_list[i])) - assert(self.nodes[0].getrawtransaction(segwit_tx_list[i]) != self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) - assert(self.nodes[1].getrawtransaction(segwit_tx_list[i]) == self.nodes[2].gettransaction(segwit_tx_list[i])["hex"]) - assert(self.nodes[0].getrawtransaction(segwit_tx_list[i]) == bytes_to_hex_str(tx.serialize_without_witness())) + assert self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False) + assert self.nodes[1].getblock(blockhash, False) == self.nodes[2].getblock(blockhash, False) + + for tx_id in segwit_tx_list: + tx = FromHex(CTransaction(), self.nodes[2].gettransaction(tx_id)["hex"]) + assert self.nodes[2].getrawtransaction(tx_id, False, blockhash) != self.nodes[0].getrawtransaction(tx_id, False, blockhash) + assert self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].getrawtransaction(tx_id, False, blockhash) + assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) != self.nodes[2].gettransaction(tx_id)["hex"] + assert self.nodes[1].getrawtransaction(tx_id, False, blockhash) == self.nodes[2].gettransaction(tx_id)["hex"] + assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) == tx.serialize_without_witness().hex() self.log.info("Verify witness txs without witness data are invalid after the fork") self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch) (code 64)', wit_ids[NODE_2][WIT_V0][2], sign=False) @@ -199,11 +198,11 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(tmpl['sizelimit'] >= 3999577) # actual maximum size is lower due to minimum mandatory non-witness data - assert(tmpl['weightlimit'] == 4000000) - assert(tmpl['sigoplimit'] == 80000) - assert(tmpl['transactions'][0]['txid'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 8) + assert tmpl['sizelimit'] >= 3999577 # actual maximum size is lower due to minimum mandatory non-witness data + assert tmpl['weightlimit'] == 4000000 + assert tmpl['sigoplimit'] == 80000 + assert tmpl['transactions'][0]['txid'] == txid + assert tmpl['transactions'][0]['sigops'] == 8 self.nodes[0].generate(1) # Mine a block to clear the gbt cache @@ -215,8 +214,8 @@ class SegWitTest(BitcoinTestFramework): txid1 = send_to_witness(1, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[0], False, Decimal("49.996")) hex_tx = self.nodes[0].gettransaction(txid)['hex'] tx = FromHex(CTransaction(), hex_tx) - assert(tx.wit.is_null()) # This should not be a segwit input - assert(txid1 in self.nodes[0].getrawmempool()) + assert tx.wit.is_null() # This should not be a segwit input + assert txid1 in self.nodes[0].getrawmempool() # Now create tx2, which will spend from txid1. tx = CTransaction() @@ -225,7 +224,7 @@ class SegWitTest(BitcoinTestFramework): tx2_hex = self.nodes[0].signrawtransactionwithwallet(ToHex(tx))['hex'] txid2 = self.nodes[0].sendrawtransaction(tx2_hex) tx = FromHex(CTransaction(), tx2_hex) - assert(not tx.wit.is_null()) + assert not tx.wit.is_null() # Now create tx3, which will spend from txid2 tx = CTransaction() @@ -233,15 +232,15 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(int(49.95 * COIN), CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))) # Huge fee tx.calc_sha256() txid3 = self.nodes[0].sendrawtransaction(ToHex(tx)) - assert(tx.wit.is_null()) - assert(txid3 in self.nodes[0].getrawmempool()) + assert tx.wit.is_null() + assert txid3 in self.nodes[0].getrawmempool() # Check that getblocktemplate includes all transactions. template = self.nodes[0].getblocktemplate({"rules": ["segwit"]}) template_txids = [t['txid'] for t in template['transactions']] - assert(txid1 in template_txids) - assert(txid2 in template_txids) - assert(txid3 in template_txids) + assert txid1 in template_txids + assert txid2 in template_txids + assert txid3 in template_txids # Check that wtxid is properly reported in mempool entry assert_equal(int(self.nodes[0].getmempoolentry(txid3)["wtxid"], 16), tx.calc_sha256(True)) @@ -393,22 +392,22 @@ class SegWitTest(BitcoinTestFramework): v = self.nodes[0].getaddressinfo(i) if (v['isscript']): bare = hex_str_to_bytes(v['hex']) - importlist.append(bytes_to_hex_str(bare)) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(bare)]))) + importlist.append(bare.hex()) + importlist.append(CScript([OP_0, sha256(bare)]).hex()) else: pubkey = hex_str_to_bytes(v['pubkey']) p2pk = CScript([pubkey, OP_CHECKSIG]) p2pkh = CScript([OP_DUP, OP_HASH160, hash160(pubkey), OP_EQUALVERIFY, OP_CHECKSIG]) - importlist.append(bytes_to_hex_str(p2pk)) - importlist.append(bytes_to_hex_str(p2pkh)) - importlist.append(bytes_to_hex_str(CScript([OP_0, hash160(pubkey)]))) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(p2pk)]))) - importlist.append(bytes_to_hex_str(CScript([OP_0, sha256(p2pkh)]))) + importlist.append(p2pk.hex()) + importlist.append(p2pkh.hex()) + importlist.append(CScript([OP_0, hash160(pubkey)]).hex()) + importlist.append(CScript([OP_0, sha256(p2pk)]).hex()) + importlist.append(CScript([OP_0, sha256(p2pkh)]).hex()) - importlist.append(bytes_to_hex_str(unsolvablep2pkh)) - importlist.append(bytes_to_hex_str(unsolvablep2wshp2pkh)) - importlist.append(bytes_to_hex_str(op1)) - importlist.append(bytes_to_hex_str(p2wshop1)) + importlist.append(unsolvablep2pkh.hex()) + importlist.append(unsolvablep2wshp2pkh.hex()) + importlist.append(op1.hex()) + importlist.append(p2wshop1.hex()) for i in importlist: # import all generated addresses. The wallet already has the private keys for some of these, so catch JSON RPC @@ -536,9 +535,9 @@ class SegWitTest(BitcoinTestFramework): for i in script_list: tx.vout.append(CTxOut(10000000, i)) tx.rehash() - signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex'] + signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] txid = self.nodes[0].sendrawtransaction(signresults, True) - self.nodes[0].generate(1) + txs_mined[txid] = self.nodes[0].generate(1)[0] sync_blocks(self.nodes) watchcount = 0 spendcount = 0 @@ -581,14 +580,14 @@ class SegWitTest(BitcoinTestFramework): tx = CTransaction() for i in txids: txtmp = CTransaction() - txraw = self.nodes[0].getrawtransaction(i) + txraw = self.nodes[0].getrawtransaction(i, 0, txs_mined[i]) f = BytesIO(hex_str_to_bytes(txraw)) txtmp.deserialize(f) for j in range(len(txtmp.vout)): tx.vin.append(CTxIn(COutPoint(int('0x' + i, 0), j))) tx.vout.append(CTxOut(0, CScript())) tx.rehash() - signresults = self.nodes[0].signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize_without_witness()))['hex'] + signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] self.nodes[0].sendrawtransaction(signresults, True) self.nodes[0].generate(1) sync_blocks(self.nodes) diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 88df61cabc..0713925141 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test version bits warning system. @@ -22,7 +22,6 @@ VB_TOP_BITS = 0x20000000 VB_UNKNOWN_BIT = 27 # Choose a bit unassigned to any deployment VB_UNKNOWN_VERSION = VB_TOP_BITS | (1 << VB_UNKNOWN_BIT) -WARN_UNKNOWN_RULES_MINED = "Unknown block versions being mined! It's possible unknown rules are in effect" WARN_UNKNOWN_RULES_ACTIVE = "unknown new rules activated (versionbit {})".format(VB_UNKNOWN_BIT) VB_PATTERN = re.compile("Warning: unknown new rules activated.*versionbit") @@ -75,18 +74,13 @@ class VersionBitsWarningTest(BitcoinTestFramework): node.generatetoaddress(VB_PERIOD - VB_THRESHOLD + 1, node_deterministic_address) # Check that we're not getting any versionbit-related errors in get*info() - assert(not VB_PATTERN.match(node.getmininginfo()["warnings"])) - assert(not VB_PATTERN.match(node.getnetworkinfo()["warnings"])) + assert not VB_PATTERN.match(node.getmininginfo()["warnings"]) + assert not VB_PATTERN.match(node.getnetworkinfo()["warnings"]) - self.log.info("Check that there is a warning if >50 blocks in the last 100 were an unknown version") # Build one period of blocks with VB_THRESHOLD blocks signaling some unknown bit self.send_blocks_with_version(node.p2p, VB_THRESHOLD, VB_UNKNOWN_VERSION) node.generatetoaddress(VB_PERIOD - VB_THRESHOLD, node_deterministic_address) - # Check that get*info() shows the 51/100 unknown block version error. - assert(WARN_UNKNOWN_RULES_MINED in node.getmininginfo()["warnings"]) - assert(WARN_UNKNOWN_RULES_MINED in node.getnetworkinfo()["warnings"]) - self.log.info("Check that there is a warning if previous VB_BLOCKS have >=VB_THRESHOLD blocks with unknown versionbits version.") # Mine a period worth of expected blocks so the generic block-version warning # is cleared. This will move the versionbit state to ACTIVE. @@ -101,8 +95,8 @@ class VersionBitsWarningTest(BitcoinTestFramework): # Generating one more block will be enough to generate an error. node.generatetoaddress(1, node_deterministic_address) # Check that get*info() shows the versionbits unknown rules warning - assert(WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"]) - assert(WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"]) + assert WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"] + assert WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"] # Check that the alert file shows the versionbits unknown rules warning wait_until(lambda: self.versionbits_in_alert_file(), timeout=60) diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index aed439339f..15b7ba9ae1 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" @@ -16,7 +16,7 @@ class TestBitcoinCli(BitcoinTestFramework): """Main test logic""" cli_response = self.nodes[0].cli("-version").send_cli() - assert("Bitcoin Core RPC client version" in cli_response) + assert "Bitcoin Core RPC client version" in cli_response self.log.info("Compare responses from getwalletinfo RPC and `bitcoin-cli getwalletinfo`") if self.is_wallet_compiled(): @@ -57,16 +57,14 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['version'], network_info['version']) assert_equal(cli_get_info['protocolversion'], network_info['protocolversion']) - if self.is_wallet_compiled(): - assert_equal(cli_get_info['walletversion'], wallet_info['walletversion']) - assert_equal(cli_get_info['balance'], wallet_info['balance']) assert_equal(cli_get_info['blocks'], blockchain_info['blocks']) assert_equal(cli_get_info['timeoffset'], network_info['timeoffset']) assert_equal(cli_get_info['connections'], network_info['connections']) assert_equal(cli_get_info['proxy'], network_info['networks'][0]['proxy']) assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) - assert_equal(cli_get_info['testnet'], blockchain_info['chain'] == "test") + assert_equal(cli_get_info['chain'], blockchain_info['chain']) if self.is_wallet_compiled(): + assert_equal(cli_get_info['walletversion'], wallet_info['walletversion']) assert_equal(cli_get_info['balance'], wallet_info['balance']) assert_equal(cli_get_info['keypoololdest'], wallet_info['keypoololdest']) assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) diff --git a/test/functional/interface_http.py b/test/functional/interface_http.py index 20889366e5..bb868d7115 100755 --- a/test/functional/interface_http.py +++ b/test/functional/interface_http.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC HTTP basics.""" @@ -30,14 +30,14 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 + assert conn.sock is not None #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 #must also response with a correct json-rpc message + assert conn.sock is not None #according to http/1.1 connection must still be open! conn.close() #same should be if we add keep-alive because this should be the std. behaviour @@ -47,14 +47,14 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 + assert conn.sock is not None #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock is not None) #according to http/1.1 connection must still be open! + assert b'"error":null' in out1 #must also response with a correct json-rpc message + assert conn.sock is not None #according to http/1.1 connection must still be open! conn.close() #now do the same with "Connection: close" @@ -64,8 +64,8 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is None) #now the connection must be closed after the response + assert b'"error":null' in out1 + assert conn.sock is None #now the connection must be closed after the response #node1 (2nd node) is running with disabled keep-alive option urlNode1 = urllib.parse.urlparse(self.nodes[1].url) @@ -76,7 +76,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) + assert b'"error":null' in out1 #node2 (third node) is running with standard keep-alive parameters which means keep-alive is on urlNode2 = urllib.parse.urlparse(self.nodes[2].url) @@ -87,8 +87,8 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.connect() conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() - assert(b'"error":null' in out1) - assert(conn.sock is not None) #connection must be closed because bitcoind should use keep-alive by default + assert b'"error":null' in out1 + assert conn.sock is not None #connection must be closed because bitcoind should use keep-alive by default # Check excessive request size conn = http.client.HTTPConnection(urlNode2.hostname, urlNode2.port) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index f850a54462..a036dfc790 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the REST API.""" @@ -22,6 +22,8 @@ from test_framework.util import ( hex_str_to_bytes, ) +from test_framework.messages import BLOCK_HEADER_SIZE + class ReqType(Enum): JSON = 1 BIN = 2 @@ -41,8 +43,7 @@ class RESTTest (BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 2 - # TODO: remove -txindex. Currently required for getrawtransaction call. - self.extra_args = [["-rest", "-txindex"], []] + self.extra_args = [["-rest"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -89,15 +90,17 @@ class RESTTest (BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) self.sync_all() - self.nodes[1].generatetoaddress(1, not_related_address) - self.sync_all() - bb_hash = self.nodes[0].getbestblockhash() - assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) - - self.log.info("Load the transaction using the /tx URI") + self.log.info("Test the /tx URI") json_obj = self.test_rest_request("/tx/{}".format(txid)) + assert_equal(json_obj['txid'], txid) + + # Check hex format response + hex_response = self.test_rest_request("/tx/{}".format(txid), req_type=ReqType.HEX, ret_type=RetType.OBJ) + assert_greater_than_or_equal(int(hex_response.getheader('content-length')), + json_obj['size']*2) + spent = (json_obj['vin'][0]['txid'], json_obj['vin'][0]['vout']) # get the vin to later check for utxo (should be spent by then) # get n of 0.1 outpoint n, = filter_output_indices_by_value(json_obj['vout'], Decimal('0.1')) @@ -105,9 +108,14 @@ class RESTTest (BitcoinTestFramework): self.log.info("Query an unspent TXO using the /getutxos URI") - json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending)) + self.nodes[1].generatetoaddress(1, not_related_address) + self.sync_all() + bb_hash = self.nodes[0].getbestblockhash() + + assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) # Check chainTip response + json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending)) assert_equal(json_obj['chaintipHash'], bb_hash) # Make sure there is one utxo @@ -144,7 +152,7 @@ class RESTTest (BitcoinTestFramework): bin_response = self.test_rest_request("/getutxos", http_method='POST', req_type=ReqType.BIN, body=bin_request, ret_type=RetType.BYTES) output = BytesIO(bin_response) chain_height, = unpack("i", output.read(4)) - response_hash = binascii.hexlify(output.read(32)[::-1]).decode('ascii') + response_hash = output.read(32)[::-1].hex() assert_equal(bb_hash, response_hash) # check if getutxo's chaintip during calculation was fine assert_equal(chain_height, 102) # chain height must be 102 @@ -214,26 +222,26 @@ class RESTTest (BitcoinTestFramework): # Check binary format response = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.BIN, ret_type=RetType.OBJ) - assert_greater_than(int(response.getheader('content-length')), 80) + assert_greater_than(int(response.getheader('content-length')), BLOCK_HEADER_SIZE) response_bytes = response.read() # Compare with block header response_header = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.BIN, ret_type=RetType.OBJ) - assert_equal(int(response_header.getheader('content-length')), 80) + assert_equal(int(response_header.getheader('content-length')), BLOCK_HEADER_SIZE) response_header_bytes = response_header.read() - assert_equal(response_bytes[:80], response_header_bytes) + assert_equal(response_bytes[:BLOCK_HEADER_SIZE], response_header_bytes) # Check block hex format response_hex = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ) - assert_greater_than(int(response_hex.getheader('content-length')), 160) + assert_greater_than(int(response_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2) response_hex_bytes = response_hex.read().strip(b'\n') assert_equal(binascii.hexlify(response_bytes), response_hex_bytes) # Compare with hex block header response_header_hex = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ) - assert_greater_than(int(response_header_hex.getheader('content-length')), 160) - response_header_hex_bytes = response_header_hex.read(160) - assert_equal(binascii.hexlify(response_bytes[:80]), response_header_hex_bytes) + assert_greater_than(int(response_header_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2) + response_header_hex_bytes = response_header_hex.read(BLOCK_HEADER_SIZE*2) + assert_equal(binascii.hexlify(response_bytes[:BLOCK_HEADER_SIZE]), response_header_hex_bytes) # Check json format block_json_obj = self.test_rest_request("/block/{}".format(bb_hash)) @@ -244,7 +252,7 @@ class RESTTest (BitcoinTestFramework): resp_hex = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.HEX, ret_type=RetType.OBJ) assert_equal(resp_hex.read().decode('utf-8').rstrip(), bb_hash) resp_bytes = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.BIN, ret_type=RetType.BYTES) - blockhash = binascii.hexlify(resp_bytes[::-1]).decode('utf-8') + blockhash = resp_bytes[::-1].hex() assert_equal(blockhash, bb_hash) # Check invalid blockhashbyheight requests @@ -272,17 +280,6 @@ class RESTTest (BitcoinTestFramework): json_obj = self.test_rest_request("/headers/5/{}".format(bb_hash)) assert_equal(len(json_obj), 5) # now we should have 5 header objects - self.log.info("Test the /tx URI") - - tx_hash = block_json_obj['tx'][0]['txid'] - json_obj = self.test_rest_request("/tx/{}".format(tx_hash)) - assert_equal(json_obj['txid'], tx_hash) - - # Check hex format response - hex_response = self.test_rest_request("/tx/{}".format(tx_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ) - assert_greater_than_or_equal(int(hex_response.getheader('content-length')), - json_obj['size']*2) - self.log.info("Test tx inclusion in the /mempool and /block URIs") # Make 3 tx and mine them on node 1 diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index b6955d4492..67ab1f0130 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests some generic aspects of the RPC interface.""" diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 94fea37090..8e58c85c15 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ notification interface.""" @@ -10,7 +10,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import CTransaction from test_framework.util import ( assert_equal, - bytes_to_hex_str, hash256, ) from io import BytesIO @@ -94,17 +93,17 @@ class ZMQTest (BitcoinTestFramework): tx = CTransaction() tx.deserialize(BytesIO(hex)) tx.calc_sha256() - assert_equal(tx.hash, bytes_to_hex_str(txid)) + assert_equal(tx.hash, txid.hex()) # Should receive the generated block hash. - hash = bytes_to_hex_str(self.hashblock.receive()) + hash = self.hashblock.receive().hex() assert_equal(genhashes[x], hash) # The block should only have the coinbase txid. - assert_equal([bytes_to_hex_str(txid)], self.nodes[1].getblock(hash)["tx"]) + assert_equal([txid.hex()], self.nodes[1].getblock(hash)["tx"]) # Should receive the generated raw block. block = self.rawblock.receive() - assert_equal(genhashes[x], bytes_to_hex_str(hash256(block[:80]))) + assert_equal(genhashes[x], hash256(block[:80]).hex()) if self.is_wallet_compiled(): self.log.info("Wait for tx from second node") @@ -113,11 +112,11 @@ class ZMQTest (BitcoinTestFramework): # Should receive the broadcasted txid. txid = self.hashtx.receive() - assert_equal(payment_txid, bytes_to_hex_str(txid)) + assert_equal(payment_txid, txid.hex()) # Should receive the broadcasted raw transaction. hex = self.rawtx.receive() - assert_equal(payment_txid, bytes_to_hex_str(hash256(hex))) + assert_equal(payment_txid, hash256(hex).hex()) self.log.info("Test the getzmqnotifications RPC") diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index e2a219b85a..fc81119050 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool acceptance of raw transactions.""" @@ -27,9 +27,7 @@ from test_framework.script import ( from test_framework.util import ( assert_equal, assert_raises_rpc_error, - bytes_to_hex_str, hex_str_to_bytes, - wait_until, ) @@ -38,7 +36,6 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [[ '-txindex', - '-reindex', # Need reindex for txindex '-acceptnonstdtxn=0', # Try to mimic main-net ]] * self.num_nodes @@ -56,7 +53,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('Start with empty mempool, and 200 blocks') self.mempool_size = 0 - wait_until(lambda: node.getblockcount() == 200) + assert_equal(node.getblockcount(), 200) assert_equal(node.getmempoolinfo()['size'], self.mempool_size) coins = node.listunspent() @@ -103,7 +100,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_final))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': True}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], allowhighfees=True, ) node.sendrawtransaction(hexstring=raw_tx_final, allowhighfees=True) @@ -121,7 +118,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vout[0].nValue -= int(fee * COIN) # Double the fee tx.vin[0].nSequence = BIP125_SEQUENCE_NUMBER + 1 # Now, opt out of RBF - raw_tx_0 = node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex'] + raw_tx_0 = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) txid_0 = tx.rehash() self.check_mempool_result( @@ -131,14 +128,14 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A transaction that conflicts with an unconfirmed tx') # Send the transaction that replaces the mempool transaction and opts out of replaceability - node.sendrawtransaction(hexstring=bytes_to_hex_str(tx.serialize()), allowhighfees=True) + node.sendrawtransaction(hexstring=tx.serialize().hex(), allowhighfees=True) # take original raw_tx_0 tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vout[0].nValue -= int(4 * fee * COIN) # Set more fee # skip re-signing the tx self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '18: txn-mempool-conflict'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], allowhighfees=True, ) @@ -148,13 +145,13 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # skip re-signing the tx self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'missing-inputs'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with missing inputs, that existed once in the past') tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_0))) tx.vin[0].prevout.n = 1 # Set vout to 1, to spend the other outpoint (49 coins) of the in-chain-tx we want to double spend - raw_tx_1 = node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex'] + raw_tx_1 = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] txid_1 = node.sendrawtransaction(hexstring=raw_tx_1, allowhighfees=True) # Now spend both to "clearly hide" the outputs, ie. remove the coins from the utxo set by spending them raw_tx_spend_both = node.signrawtransactionwithwallet(node.createrawtransaction( @@ -186,17 +183,17 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # Reference tx should be valid on itself self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': True}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with no outputs') tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout = [] # Skip re-signing the transaction for context independent checks from now on - # tx.deserialize(BytesIO(hex_str_to_bytes(node.signrawtransactionwithwallet(bytes_to_hex_str(tx.serialize()))['hex']))) + # tx.deserialize(BytesIO(hex_str_to_bytes(node.signrawtransactionwithwallet(tx.serialize().hex())['hex']))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-empty'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A really large transaction') @@ -204,7 +201,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_BASE_SIZE / len(tx.vin[0].serialize())) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-oversize'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with negative output value') @@ -212,7 +209,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue *= -1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-negative'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with too large output value') @@ -220,7 +217,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue = 21000000 * COIN + 1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-vout-toolarge'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with too large sum of output values') @@ -229,7 +226,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout[0].nValue = 21000000 * COIN self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-txouttotal-toolarge'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction with duplicate inputs') @@ -237,7 +234,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vin = [tx.vin[0]] * 2 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: bad-txns-inputs-duplicate'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A coinbase transaction') @@ -246,7 +243,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_coinbase_spent))) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '16: coinbase'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('Some nonstandard transactions') @@ -254,19 +251,19 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.nVersion = 3 # A version currently non-standard self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: version'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].scriptPubKey = CScript([OP_0]) # Some non-standard script self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: scriptpubkey'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vin[0].scriptSig = CScript([OP_HASH160]) # Some not-pushonly scriptSig self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: scriptsig-not-pushonly'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) output_p2sh_burn = CTxOut(nValue=540, scriptPubKey=CScript([OP_HASH160, hash160(b'burn'), OP_EQUAL])) @@ -274,21 +271,21 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.vout = [output_p2sh_burn] * num_scripts self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: tx-size'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0] = output_p2sh_burn tx.vout[0].nValue -= 1 # Make output smaller, such that it is dust for our policy self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: dust'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_reference))) tx.vout[0].scriptPubKey = CScript([OP_RETURN, b'\xff']) tx.vout = [tx.vout[0]] * 2 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: multi-op-return'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A timelocked transaction') @@ -297,7 +294,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): tx.nLockTime = node.getblockcount() + 1 self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: non-final'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], ) self.log.info('A transaction that is locked by BIP68 sequence logic') @@ -306,7 +303,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): # Can skip re-signing the tx because of early rejection self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': '64: non-BIP68-final'}], - rawtxs=[bytes_to_hex_str(tx.serialize())], + rawtxs=[tx.serialize().hex()], allowhighfees=True, ) diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index c0918893cd..351b27e94a 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool limiting together/eviction with the wallet.""" @@ -47,9 +47,9 @@ class MempoolLimitTest(BitcoinTestFramework): txids[i] = create_lots_of_big_transactions(self.nodes[0], txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) self.log.info('The tx should be evicted by now') - assert(txid not in self.nodes[0].getrawmempool()) + assert txid not in self.nodes[0].getrawmempool() txdata = self.nodes[0].gettransaction(txid) - assert(txdata['confirmations'] == 0) #confirmation should still be 0 + assert txdata['confirmations'] == 0 #confirmation should still be 0 self.log.info('Check that mempoolminfee is larger than minrelytxfee') assert_equal(self.nodes[0].getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 9336547a6b..7cee215d54 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descendant package tracking code.""" @@ -33,7 +33,7 @@ class MempoolPackagesTest(BitcoinTestFramework): signedtx = node.signrawtransactionwithwallet(rawtx) txid = node.sendrawtransaction(signedtx['hex']) fulltx = node.getrawtransaction(txid, 1) - assert(len(fulltx['vout']) == num_outputs) # make sure we didn't generate a change output + assert len(fulltx['vout']) == num_outputs # make sure we didn't generate a change output return (txid, send_value) def run_test(self): @@ -125,13 +125,13 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(len(v_ancestors), len(chain)-1) for x in v_ancestors.keys(): assert_equal(mempool[x], v_ancestors[x]) - assert(chain[-1] not in v_ancestors.keys()) + assert chain[-1] not in v_ancestors.keys() v_descendants = self.nodes[0].getmempooldescendants(chain[0], True) assert_equal(len(v_descendants), len(chain)-1) for x in v_descendants.keys(): assert_equal(mempool[x], v_descendants[x]) - assert(chain[0] not in v_descendants.keys()) + assert chain[0] not in v_descendants.keys() # Check that ancestor modified fees includes fee deltas from # prioritisetransaction diff --git a/test/functional/mempool_resurrect.py b/test/functional/mempool_resurrect.py index 845beb551e..187c9026f6 100755 --- a/test/functional/mempool_resurrect.py +++ b/test/functional/mempool_resurrect.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test resurrection of mined transactions when the blockchain is re-organized.""" @@ -45,7 +45,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set()) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] > 0) + assert tx["confirmations"] > 0 # Use invalidateblock to re-org back for node in self.nodes: @@ -55,7 +55,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set(spends1_id+spends2_id)) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] == 0) + assert tx["confirmations"] == 0 # Generate another block, they should all get mined self.nodes[0].generate(1) @@ -63,7 +63,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(self.nodes[0].getrawmempool()), set()) for txid in spends1_id+spends2_id: tx = self.nodes[0].gettransaction(txid) - assert(tx["confirmations"] > 0) + assert tx["confirmations"] > 0 if __name__ == '__main__': diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index 661d9f4c97..788aabc192 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mining RPCs @@ -11,10 +11,14 @@ import copy from decimal import Decimal -from test_framework.blocktools import create_coinbase +from test_framework.blocktools import ( + create_coinbase, + TIME_GENESIS_BLOCK, +) from test_framework.messages import ( CBlock, CBlockHeader, + BLOCK_HEADER_SIZE ) from test_framework.mininode import ( P2PDataStore, @@ -23,37 +27,51 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - bytes_to_hex_str as b2x, + connect_nodes_bi, ) from test_framework.script import CScriptNum + def assert_template(node, block, expect, rehash=True): if rehash: block.hashMerkleRoot = block.calc_merkle_root() - rsp = node.getblocktemplate(template_request={'data': b2x(block.serialize()), 'mode': 'proposal', 'rules': ['segwit']}) + rsp = node.getblocktemplate(template_request={'data': block.serialize().hex(), 'mode': 'proposal', 'rules': ['segwit']}) assert_equal(rsp, expect) class MiningTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - self.setup_clean_chain = False + self.setup_clean_chain = True + + def mine_chain(self): + self.log.info('Create some old blocks') + for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): + self.nodes[0].setmocktime(t) + self.nodes[0].generate(1) + mining_info = self.nodes[0].getmininginfo() + assert_equal(mining_info['blocks'], 200) + assert_equal(mining_info['currentblocktx'], 0) + assert_equal(mining_info['currentblockweight'], 4000) + self.restart_node(0) + connect_nodes_bi(self.nodes, 0, 1) def run_test(self): + self.mine_chain() node = self.nodes[0] def assert_submitblock(block, result_str_1, result_str_2=None): block.solve() result_str_2 = result_str_2 or 'duplicate-invalid' - assert_equal(result_str_1, node.submitblock(hexdata=b2x(block.serialize()))) - assert_equal(result_str_2, node.submitblock(hexdata=b2x(block.serialize()))) + assert_equal(result_str_1, node.submitblock(hexdata=block.serialize().hex())) + assert_equal(result_str_2, node.submitblock(hexdata=block.serialize().hex())) self.log.info('getmininginfo') mining_info = node.getmininginfo() assert_equal(mining_info['blocks'], 200) assert_equal(mining_info['chain'], 'regtest') - assert_equal(mining_info['currentblocktx'], 0) - assert_equal(mining_info['currentblockweight'], 0) + assert 'currentblocktx' not in mining_info + assert 'currentblockweight' not in mining_info assert_equal(mining_info['difficulty'], Decimal('4.656542373906925E-10')) assert_equal(mining_info['networkhashps'], Decimal('0.003333333333333334')) assert_equal(mining_info['pooledtx'], 0) @@ -93,7 +111,7 @@ class MiningTest(BitcoinTestFramework): assert_template(node, block, None) self.log.info("submitblock: Test block decode failure") - assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, b2x(block.serialize()[:-15])) + assert_raises_rpc_error(-22, "Block decode failed", node.submitblock, block.serialize()[:-15].hex()) self.log.info("getblocktemplate: Test bad input hash for coinbase transaction") bad_block = copy.deepcopy(block) @@ -102,10 +120,10 @@ class MiningTest(BitcoinTestFramework): assert_template(node, bad_block, 'bad-cb-missing') self.log.info("submitblock: Test invalid coinbase transaction") - assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, b2x(bad_block.serialize())) + assert_raises_rpc_error(-22, "Block does not start with a coinbase", node.submitblock, bad_block.serialize().hex()) self.log.info("getblocktemplate: Test truncated final transaction") - assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(block.serialize()[:-1]), 'mode': 'proposal', 'rules': ['segwit']}) + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': block.serialize()[:-1].hex(), 'mode': 'proposal', 'rules': ['segwit']}) self.log.info("getblocktemplate: Test duplicate transaction") bad_block = copy.deepcopy(block) @@ -131,11 +149,10 @@ class MiningTest(BitcoinTestFramework): self.log.info("getblocktemplate: Test bad tx count") # The tx count is immediately after the block header - TX_COUNT_OFFSET = 80 bad_block_sn = bytearray(block.serialize()) - assert_equal(bad_block_sn[TX_COUNT_OFFSET], 1) - bad_block_sn[TX_COUNT_OFFSET] += 1 - assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': b2x(bad_block_sn), 'mode': 'proposal', 'rules': ['segwit']}) + assert_equal(bad_block_sn[BLOCK_HEADER_SIZE], 1) + bad_block_sn[BLOCK_HEADER_SIZE] += 1 + assert_raises_rpc_error(-22, "Block decode failed", node.getblocktemplate, {'data': bad_block_sn.hex(), 'mode': 'proposal', 'rules': ['segwit']}) self.log.info("getblocktemplate: Test bad bits") bad_block = copy.deepcopy(block) @@ -164,9 +181,9 @@ class MiningTest(BitcoinTestFramework): assert_submitblock(bad_block, 'prev-blk-not-found', 'prev-blk-not-found') self.log.info('submitheader tests') - assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='xx' * 80)) - assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='ff' * 78)) - assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata='ff' * 80)) + assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='xx' * BLOCK_HEADER_SIZE)) + assert_raises_rpc_error(-22, 'Block header decode failed', lambda: node.submitheader(hexdata='ff' * (BLOCK_HEADER_SIZE-2))) + assert_raises_rpc_error(-25, 'Must submit previous header', lambda: node.submitheader(hexdata=super(CBlock, bad_block).serialize().hex())) block.nTime += 1 block.solve() @@ -175,23 +192,23 @@ class MiningTest(BitcoinTestFramework): return {'hash': b_hash, 'height': 202, 'branchlen': branchlen, 'status': status} assert chain_tip(block.hash) not in node.getchaintips() - node.submitheader(hexdata=b2x(block.serialize())) + node.submitheader(hexdata=block.serialize().hex()) assert chain_tip(block.hash) in node.getchaintips() - node.submitheader(hexdata=b2x(CBlockHeader(block).serialize())) # Noop + node.submitheader(hexdata=CBlockHeader(block).serialize().hex()) # Noop assert chain_tip(block.hash) in node.getchaintips() bad_block_root = copy.deepcopy(block) bad_block_root.hashMerkleRoot += 2 bad_block_root.solve() assert chain_tip(bad_block_root.hash) not in node.getchaintips() - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) assert chain_tip(bad_block_root.hash) in node.getchaintips() # Should still reject invalid blocks, even if we have the header: - assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot') - assert_equal(node.submitblock(hexdata=b2x(bad_block_root.serialize())), 'bad-txnmrklroot') + assert_equal(node.submitblock(hexdata=bad_block_root.serialize().hex()), 'bad-txnmrklroot') + assert_equal(node.submitblock(hexdata=bad_block_root.serialize().hex()), 'bad-txnmrklroot') assert chain_tip(bad_block_root.hash) in node.getchaintips() # We know the header for this invalid block, so should just return early without error: - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) assert chain_tip(bad_block_root.hash) in node.getchaintips() bad_block_lock = copy.deepcopy(block) @@ -199,19 +216,19 @@ class MiningTest(BitcoinTestFramework): bad_block_lock.vtx[0].rehash() bad_block_lock.hashMerkleRoot = bad_block_lock.calc_merkle_root() bad_block_lock.solve() - assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'bad-txns-nonfinal') - assert_equal(node.submitblock(hexdata=b2x(bad_block_lock.serialize())), 'duplicate-invalid') + assert_equal(node.submitblock(hexdata=bad_block_lock.serialize().hex()), 'bad-txns-nonfinal') + assert_equal(node.submitblock(hexdata=bad_block_lock.serialize().hex()), 'duplicate-invalid') # Build a "good" block on top of the submitted bad block bad_block2 = copy.deepcopy(block) bad_block2.hashPrevBlock = bad_block_lock.sha256 bad_block2.solve() - assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize()))) + assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=CBlockHeader(bad_block2).serialize().hex())) # Should reject invalid header right away bad_block_time = copy.deepcopy(block) bad_block_time.nTime = 1 bad_block_time.solve() - assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize()))) + assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=CBlockHeader(bad_block_time).serialize().hex())) # Should ask for the block from a p2p node, if they announce the header as well: node.add_p2p_connection(P2PDataStore()) @@ -222,11 +239,11 @@ class MiningTest(BitcoinTestFramework): # Building a few blocks should give the same results node.generatetoaddress(10, node.get_deterministic_priv_key().address) - assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block_time).serialize()))) - assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=b2x(CBlockHeader(bad_block2).serialize()))) - node.submitheader(hexdata=b2x(CBlockHeader(block).serialize())) - node.submitheader(hexdata=b2x(CBlockHeader(bad_block_root).serialize())) - assert_equal(node.submitblock(hexdata=b2x(block.serialize())), 'duplicate') # valid + assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=CBlockHeader(bad_block_time).serialize().hex())) + assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=CBlockHeader(bad_block2).serialize().hex())) + node.submitheader(hexdata=CBlockHeader(block).serialize().hex()) + node.submitheader(hexdata=CBlockHeader(bad_block_root).serialize().hex()) + assert_equal(node.submitblock(hexdata=block.serialize().hex()), 'duplicate') # valid if __name__ == '__main__': diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index 72cde8e811..445ec124ce 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test longpolling with getblocktemplate.""" @@ -38,27 +38,27 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): longpollid = template['longpollid'] # longpollid should not change between successive invocations if nothing else happens template2 = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(template2['longpollid'] == longpollid) + assert template2['longpollid'] == longpollid # Test 1: test that the longpolling wait if we do nothing thr = LongpollThread(self.nodes[0]) thr.start() # check that thread still lives thr.join(5) # wait 5 seconds or until thread exits - assert(thr.is_alive()) + assert thr.is_alive() # Test 2: test that longpoll will terminate if another node generates a block self.nodes[1].generate(1) # generate a block on another node # check that thread will exit now that new transaction entered mempool thr.join(5) # wait 5 seconds or until thread exits - assert(not thr.is_alive()) + assert not thr.is_alive() # Test 3: test that longpoll will terminate if we generate a block ourselves thr = LongpollThread(self.nodes[0]) thr.start() self.nodes[0].generate(1) # generate a block on another node thr.join(5) # wait 5 seconds or until thread exits - assert(not thr.is_alive()) + assert not thr.is_alive() # Test 4: test that introducing a new transaction into the mempool will terminate the longpoll thr = LongpollThread(self.nodes[0]) @@ -69,7 +69,7 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): (txid, txhex, fee) = random_transaction(self.nodes, Decimal("1.1"), min_relay_fee, Decimal("0.001"), 20) # after one minute, every 10 seconds the mempool is probed, so in 80 seconds it should have returned thr.join(60 + 20) - assert(not thr.is_alive()) + assert not thr.is_alive() if __name__ == '__main__': GetBlockTemplateLPTest().main() diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index ca4b621a78..687a5b527b 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the prioritisetransaction mining RPC.""" @@ -63,9 +63,9 @@ class PrioritiseTransactionTest(BitcoinTestFramework): sizes = [0, 0, 0] for i in range(3): for j in txids[i]: - assert(j in mempool) + assert j in mempool sizes[i] += mempool[j]['size'] - assert(sizes[i] > MAX_BLOCK_BASE_SIZE) # Fail => raise utxo_count + assert sizes[i] > MAX_BLOCK_BASE_SIZE # Fail => raise utxo_count # add a fee delta to something in the cheapest bucket and make sure it gets mined # also check that a different entry in the cheapest bucket is NOT mined @@ -75,8 +75,8 @@ class PrioritiseTransactionTest(BitcoinTestFramework): mempool = self.nodes[0].getrawmempool() self.log.info("Assert that prioritised transaction was mined") - assert(txids[0][0] not in mempool) - assert(txids[0][1] in mempool) + assert txids[0][0] not in mempool + assert txids[0][1] in mempool high_fee_tx = None for x in txids[2]: @@ -84,7 +84,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): high_fee_tx = x # Something high-fee should have been mined! - assert(high_fee_tx is not None) + assert high_fee_tx is not None # Add a prioritisation before a tx is in the mempool (de-prioritising a # high-fee transaction so that it's now low fee). @@ -95,7 +95,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # Check to make sure our high fee rate tx is back in the mempool mempool = self.nodes[0].getrawmempool() - assert(high_fee_tx in mempool) + assert high_fee_tx in mempool # Now verify the modified-high feerate transaction isn't mined before # the other high fee transactions. Keep mining until our mempool has @@ -107,14 +107,14 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # transactions should have been. mempool = self.nodes[0].getrawmempool() self.log.info("Assert that de-prioritised transaction is still in mempool") - assert(high_fee_tx in mempool) + assert high_fee_tx in mempool for x in txids[2]: if (x != high_fee_tx): - assert(x not in mempool) + assert x not in mempool # Create a free transaction. Should be rejected. utxo_list = self.nodes[0].listunspent() - assert(len(utxo_list) > 0) + assert len(utxo_list) > 0 utxo = utxo_list[0] inputs = [] @@ -127,7 +127,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # This will raise an exception due to min relay fee not being met assert_raises_rpc_error(-26, "min relay fee not met", self.nodes[0].sendrawtransaction, tx_hex) - assert(tx_id not in self.nodes[0].getrawmempool()) + assert tx_id not in self.nodes[0].getrawmempool() # This is a less than 1000-byte transaction, so just set the fee # to be the minimum for a 1000-byte transaction and check that it is @@ -136,7 +136,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.log.info("Assert that prioritised free transaction is accepted to mempool") assert_equal(self.nodes[0].sendrawtransaction(tx_hex), tx_id) - assert(tx_id in self.nodes[0].getrawmempool()) + assert tx_id in self.nodes[0].getrawmempool() # Test that calling prioritisetransaction is sufficient to trigger # getblocktemplate to (eventually) return a new block. @@ -147,7 +147,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.nodes[0].setmocktime(mock_time+10) new_template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert(template != new_template) + assert template != new_template if __name__ == '__main__': PrioritiseTransactionTest().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index f0dee59b5c..82a14af9da 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test compact blocks (BIP 152). @@ -120,7 +120,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Doesn't matter which node we use, just use node0. block = self.build_block_on_tip(self.nodes[0]) self.test_node.send_and_ping(msg_block(block)) - assert(int(self.nodes[0].getbestblockhash(), 16) == block.sha256) + assert int(self.nodes[0].getbestblockhash(), 16) == block.sha256 self.nodes[0].generate(100) total_value = block.vtx[0].vout[0].nValue @@ -167,7 +167,7 @@ class CompactBlocksTest(BitcoinTestFramework): peer.clear_block_announcement() block_hash = int(node.generate(1)[0], 16) peer.wait_for_block_announcement(block_hash, timeout=30) - assert(peer.block_announced) + assert peer.block_announced with mininode_lock: assert predicate(peer), ( @@ -303,7 +303,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert("cmpctblock" in test_node.last_message) + assert "cmpctblock" in test_node.last_message # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -319,7 +319,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now fetch and check the compact block header_and_shortids = None with mininode_lock: - assert("cmpctblock" in test_node.last_message) + assert "cmpctblock" in test_node.last_message # Convert the on-the-wire representation to absolute indexes header_and_shortids = HeaderAndShortIDs(test_node.last_message["cmpctblock"].header_and_shortids) self.check_compactblock_construction_from_block(version, header_and_shortids, block_hash, block) @@ -330,7 +330,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(header_and_shortids.header.sha256, block_hash) # Make sure the prefilled_txn appears to have included the coinbase - assert(len(header_and_shortids.prefilled_txn) >= 1) + assert len(header_and_shortids.prefilled_txn) >= 1 assert_equal(header_and_shortids.prefilled_txn[0].index, 0) # Check that all prefilled_txn entries match what's in the block. @@ -345,7 +345,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(wtxid, block.vtx[entry.index].calc_sha256(True)) else: # Shouldn't have received a witness - assert(entry.tx.wit.is_null()) + assert entry.tx.wit.is_null() # Check that the cmpctblock message announced all the transactions. assert_equal(len(header_and_shortids.prefilled_txn) + len(header_and_shortids.shortids), len(block.vtx)) @@ -407,7 +407,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # Expect a getblocktxn message. with mininode_lock: - assert("getblocktxn" in test_node.last_message) + assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [0]) # should be a coinbase request @@ -447,7 +447,7 @@ class CompactBlocksTest(BitcoinTestFramework): msg = msg_cmpctblock(compact_block.to_p2p()) peer.send_and_ping(msg) with mininode_lock: - assert("getblocktxn" in peer.last_message) + assert "getblocktxn" in peer.last_message absolute_indexes = peer.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, expected_result) @@ -487,7 +487,7 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_with_transactions(node, utxo, 5) self.utxos.append([block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue]) test_node.send_and_ping(msg_tx(block.vtx[1])) - assert(block.vtx[1].hash in node.getrawmempool()) + assert block.vtx[1].hash in node.getrawmempool() # Prefill 4 out of the 6 transactions, and verify that only the one # that was not in the mempool is requested. @@ -508,7 +508,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure all transactions were accepted. mempool = node.getrawmempool() for tx in block.vtx[1:]: - assert(tx.hash in mempool) + assert tx.hash in mempool # Clear out last request. with mininode_lock: @@ -519,7 +519,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_tip_after_message(node, test_node, msg_cmpctblock(comp_block.to_p2p()), block.sha256) with mininode_lock: # Shouldn't have gotten a request for any transaction - assert("getblocktxn" not in test_node.last_message) + assert "getblocktxn" not in test_node.last_message # Incorrectly responding to a getblocktxn shouldn't cause the block to be # permanently failed. @@ -537,7 +537,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Make sure all transactions were accepted. mempool = node.getrawmempool() for tx in block.vtx[1:6]: - assert(tx.hash in mempool) + assert tx.hash in mempool # Send compact block comp_block = HeaderAndShortIDs() @@ -545,7 +545,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg_cmpctblock(comp_block.to_p2p())) absolute_indexes = [] with mininode_lock: - assert("getblocktxn" in test_node.last_message) + assert "getblocktxn" in test_node.last_message absolute_indexes = test_node.last_message["getblocktxn"].block_txn_request.to_absolute() assert_equal(absolute_indexes, [6, 7, 8, 9, 10]) @@ -569,7 +569,7 @@ class CompactBlocksTest(BitcoinTestFramework): # We should receive a getdata request wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) assert_equal(len(test_node.last_message["getdata"].inv), 1) - assert(test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG) + assert test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Deliver the block @@ -606,7 +606,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(tx.sha256, block.vtx[index].sha256) if version == 1: # Witnesses should have been stripped - assert(tx.wit.is_null()) + assert tx.wit.is_null() else: # Check that the witness matches assert_equal(tx.calc_sha256(True), block.vtx[index].calc_sha256(True)) @@ -669,7 +669,7 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(x["status"], "headers-only") found = True break - assert(found) + assert found # Requesting this block via getblocktxn should silently fail # (to avoid fingerprinting attacks). @@ -707,7 +707,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Test that we don't get disconnected if we relay a compact block with valid header, # but invalid transactions. def test_invalid_tx_in_compactblock(self, node, test_node, use_segwit): - assert(len(self.utxos)) + assert len(self.utxos) utxo = self.utxos[0] block = self.build_block_with_transactions(node, utxo, 5) @@ -728,7 +728,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_and_ping(msg) # Check that the tip didn't advance - assert(int(node.getbestblockhash(), 16) is not block.sha256) + assert int(node.getbestblockhash(), 16) is not block.sha256 test_node.sync_with_ping() # Helper for enabling cb announcements @@ -743,7 +743,7 @@ class CompactBlocksTest(BitcoinTestFramework): peer.send_and_ping(msg) def test_compactblock_reconstruction_multiple_peers(self, node, stalling_peer, delivery_peer): - assert(len(self.utxos)) + assert len(self.utxos) def announce_cmpct_block(node, peer): utxo = self.utxos.pop(0) @@ -764,7 +764,7 @@ class CompactBlocksTest(BitcoinTestFramework): delivery_peer.sync_with_ping() mempool = node.getrawmempool() for tx in block.vtx[1:]: - assert(tx.hash in mempool) + assert tx.hash in mempool delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) assert_equal(int(node.getbestblockhash(), 16), block.sha256) @@ -783,7 +783,7 @@ class CompactBlocksTest(BitcoinTestFramework): cmpct_block.use_witness = True delivery_peer.send_and_ping(msg_cmpctblock(cmpct_block.to_p2p())) - assert(int(node.getbestblockhash(), 16) != block.sha256) + assert int(node.getbestblockhash(), 16) != block.sha256 msg = msg_blocktxn() msg.block_transactions.blockhash = block.sha256 @@ -882,7 +882,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Need to manually sync node0 and node1, because post-segwit activation, # node1 will not download blocks from node0. self.log.info("Syncing nodes...") - assert(self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash()) + assert self.nodes[0].getbestblockhash() != self.nodes[1].getbestblockhash() while (self.nodes[0].getblockcount() > self.nodes[1].getblockcount()): block_hash = self.nodes[0].getblockhash(self.nodes[1].getblockcount() + 1) self.nodes[1].submitblock(self.nodes[0].getblock(block_hash, False)) diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index d589519e45..b9863cf215 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of feefilter messages.""" @@ -57,7 +57,7 @@ class FeeFilterTest(BitcoinTestFramework): # Test that invs are received for all txs at feerate of 20 sat/byte node1.settxfee(Decimal("0.00020000")) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Set a filter of 15 sat/byte @@ -65,7 +65,7 @@ class FeeFilterTest(BitcoinTestFramework): # Test that txs are still being received (paying 20 sat/byte) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Change tx fee rate to 10 sat/byte and test they are no longer received @@ -82,13 +82,13 @@ class FeeFilterTest(BitcoinTestFramework): # as well. node0.settxfee(Decimal("0.00020000")) txids = [node0.sendtoaddress(node0.getnewaddress(), 1)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() # Remove fee filter and check that txs are received again self.nodes[0].p2p.send_and_ping(msg_feefilter(0)) txids = [node1.sendtoaddress(node1.getnewaddress(), 1) for x in range(3)] - assert(allInvsMatch(txids, self.nodes[0].p2p)) + assert allInvsMatch(txids, self.nodes[0].p2p) self.nodes[0].p2p.clear_invs() if __name__ == '__main__': diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index 336d34a81d..06049db54c 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test message sending before handshake completion. @@ -117,9 +117,9 @@ class P2PLeakTest(BitcoinTestFramework): wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 0) # Make sure no unexpected messages came in - assert(no_version_bannode.unexpected_msg == False) - assert(no_version_idlenode.unexpected_msg == False) - assert(no_verack_idlenode.unexpected_msg == False) + assert no_version_bannode.unexpected_msg == False + assert no_version_idlenode.unexpected_msg == False + assert no_verack_idlenode.unexpected_msg == False if __name__ == '__main__': diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 8f8e89cf15..73bfdc868c 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test segwit transactions and blocks on P2P network.""" -from binascii import hexlify import math import random import struct @@ -74,7 +73,6 @@ from test_framework.script import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - bytes_to_hex_str, connect_nodes, disconnect_nodes, get_bip9_status, @@ -317,7 +315,7 @@ class SegWitTest(BitcoinTestFramework): self.test_node.send_message(msg_witness_tx(tx)) self.test_node.sync_with_ping() # make sure the tx was processed - assert(tx.hash in self.nodes[0].getrawmempool()) + assert tx.hash in self.nodes[0].getrawmempool() # Save this transaction for later self.utxo.append(UTXO(tx.sha256, 0, 49 * 100000000)) self.nodes[0].generate(1) @@ -335,7 +333,7 @@ class SegWitTest(BitcoinTestFramework): # Verify the hash with witness differs from the txid # (otherwise our testing framework must be broken!) tx.rehash() - assert(tx.sha256 != tx.calc_sha256(with_witness=True)) + assert tx.sha256 != tx.calc_sha256(with_witness=True) # Construct a segwit-signaling block that includes the transaction. block = self.build_next_block(version=(VB_TOP_BITS | (1 << VB_WITNESS_BIT))) @@ -371,20 +369,20 @@ class SegWitTest(BitcoinTestFramework): block1.solve() self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block1, True) block2 = self.build_next_block(version=4) block2.solve() self.test_node.announce_block_and_wait_for_getdata(block2, use_header=True) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block2, True) block3 = self.build_next_block(version=(VB_TOP_BITS | (1 << 15))) block3.solve() self.test_node.announce_block_and_wait_for_getdata(block3, use_header=True) - assert(self.test_node.last_message["getdata"].inv[0].type == blocktype) + assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block3, True) # Check that we can getdata for witness blocks or regular blocks, @@ -413,8 +411,8 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, []) # This gives us a witness commitment. - assert(len(block.vtx[0].wit.vtxinwit) == 1) - assert(len(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack) == 1) + assert len(block.vtx[0].wit.vtxinwit) == 1 + assert len(block.vtx[0].wit.vtxinwit[0].scriptWitness.stack) == 1 test_witness_block(self.nodes[0], self.test_node, block, accepted=True) # Now try to retrieve it... rpc_block = self.nodes[0].getblock(block.hash, False) @@ -448,7 +446,7 @@ class SegWitTest(BitcoinTestFramework): msg.headers = [CBlockHeader(block4)] self.old_node.send_message(msg) self.old_node.announce_tx_and_wait_for_getdata(block4.vtx[0]) - assert(block4.sha256 not in self.old_node.getdataset) + assert block4.sha256 not in self.old_node.getdataset @subtest def test_v0_outputs_arent_spendable(self): @@ -537,7 +535,7 @@ class SegWitTest(BitcoinTestFramework): """Mine enough blocks for segwit's vb state to be 'started'.""" height = self.nodes[0].getblockcount() # Will need to rewrite the tests here if we are past the first period - assert(height < VB_PERIOD - 1) + assert height < VB_PERIOD - 1 # Advance to end of period, status should now be 'started' self.nodes[0].generate(VB_PERIOD - height - 1) assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') @@ -554,19 +552,19 @@ class SegWitTest(BitcoinTestFramework): # If this is a non-segwit node, we should not get a witness # commitment, nor a version bit signalling segwit. assert_equal(block_version & (1 << VB_WITNESS_BIT), 0) - assert('default_witness_commitment' not in gbt_results) + assert 'default_witness_commitment' not in gbt_results else: # For segwit-aware nodes, check the version bit and the witness # commitment are correct. - assert(block_version & (1 << VB_WITNESS_BIT) != 0) - assert('default_witness_commitment' in gbt_results) + assert block_version & (1 << VB_WITNESS_BIT) != 0 + assert 'default_witness_commitment' in gbt_results witness_commitment = gbt_results['default_witness_commitment'] # Check that default_witness_commitment is present. witness_root = CBlock.get_merkle_root([ser_uint256(0), ser_uint256(txid)]) script = get_witness_script(witness_root, 0) - assert_equal(witness_commitment, bytes_to_hex_str(script)) + assert_equal(witness_commitment, script.hex()) @subtest def advance_to_segwit_lockin(self): @@ -575,7 +573,7 @@ class SegWitTest(BitcoinTestFramework): # Advance to end of period, and verify lock-in happens at the end self.nodes[0].generate(VB_PERIOD - 1) height = self.nodes[0].getblockcount() - assert((height % VB_PERIOD) == VB_PERIOD - 2) + assert (height % VB_PERIOD) == VB_PERIOD - 2 assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'started') self.nodes[0].generate(1) assert_equal(get_bip9_status(self.nodes[0], 'segwit')['status'], 'locked_in') @@ -600,7 +598,7 @@ class SegWitTest(BitcoinTestFramework): # Verify that if a peer doesn't set nServices to include NODE_WITNESS, # the getdata is just for the non-witness portion. self.old_node.announce_tx_and_wait_for_getdata(tx) - assert(self.old_node.last_message["getdata"].inv[0].type == 1) + assert self.old_node.last_message["getdata"].inv[0].type == 1 # Since we haven't delivered the tx yet, inv'ing the same tx from # a witness transaction ought not result in a getdata. @@ -686,13 +684,13 @@ class SegWitTest(BitcoinTestFramework): if self.segwit_status != 'active': # Just check mempool acceptance, but don't add the transaction to the mempool, since witness is disallowed # in blocks and the tx is impossible to mine right now. - assert_equal(self.nodes[0].testmempoolaccept([bytes_to_hex_str(tx3.serialize_with_witness())]), [{'txid': tx3.hash, 'allowed': True}]) + assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True}]) # Create the same output as tx3, but by replacing tx tx3_out = tx3.vout[0] tx3 = tx tx3.vout = [tx3_out] tx3.rehash() - assert_equal(self.nodes[0].testmempoolaccept([bytes_to_hex_str(tx3.serialize_with_witness())]), [{'txid': tx3.hash, 'allowed': True}]) + assert_equal(self.nodes[0].testmempoolaccept([tx3.serialize_with_witness().hex()]), [{'txid': tx3.hash, 'allowed': True}]) test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) self.nodes[0].generate(1) @@ -792,7 +790,7 @@ class SegWitTest(BitcoinTestFramework): block.solve() # Test the test -- witness serialization should be different - assert(msg_witness_block(block).serialize() != msg_block(block).serialize()) + assert msg_witness_block(block).serialize() != msg_block(block).serialize() # This empty block should be valid. test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -803,7 +801,7 @@ class SegWitTest(BitcoinTestFramework): block_2.solve() # The commitment should have changed! - assert(block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1]) + assert block_2.vtx[0].vout[-1] != block.vtx[0].vout[-1] # This should also be valid. test_witness_block(self.nodes[0], self.test_node, block_2, accepted=True) @@ -850,7 +848,7 @@ class SegWitTest(BitcoinTestFramework): block_3.vtx[0].rehash() block_3.hashMerkleRoot = block_3.calc_merkle_root() block_3.rehash() - assert(len(block_3.vtx[0].vout) == 4) # 3 OP_returns + assert len(block_3.vtx[0].vout) == 4 # 3 OP_returns block_3.solve() test_witness_block(self.nodes[0], self.test_node, block_3, accepted=True) @@ -881,19 +879,19 @@ class SegWitTest(BitcoinTestFramework): block.solve() block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.append(b'a' * 5000000) - assert(get_virtual_size(block) > MAX_BLOCK_BASE_SIZE) + assert get_virtual_size(block) > MAX_BLOCK_BASE_SIZE # We can't send over the p2p network, because this is too big to relay # TODO: repeat this test with a block that can be relayed - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) - assert(self.nodes[0].getbestblockhash() != block.hash) + assert self.nodes[0].getbestblockhash() != block.hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() - assert(get_virtual_size(block) < MAX_BLOCK_BASE_SIZE) - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + assert get_virtual_size(block) < MAX_BLOCK_BASE_SIZE + self.nodes[0].submitblock(block.serialize(True).hex()) - assert(self.nodes[0].getbestblockhash() == block.hash) + assert self.nodes[0].getbestblockhash() == block.hash # Now make sure that malleating the witness reserved value doesn't # result in a block permanently marked bad. @@ -918,7 +916,7 @@ class SegWitTest(BitcoinTestFramework): # Test that witness-bearing blocks are limited at ceil(base + wit/4) <= 1MB. block = self.build_next_block() - assert(len(self.utxo) > 0) + assert len(self.utxo) > 0 # Create a P2WSH transaction. # The witness program will be a bunch of OP_2DROP's, followed by OP_TRUE. @@ -940,7 +938,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(NUM_OUTPUTS): parent_tx.vout.append(CTxOut(child_value, script_pubkey)) parent_tx.vout[0].nValue -= 50000 - assert(parent_tx.vout[0].nValue > 0) + assert parent_tx.vout[0].nValue > 0 parent_tx.rehash() child_tx = CTransaction() @@ -970,7 +968,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(vsize, MAX_BLOCK_BASE_SIZE + 1) # Make sure that our test case would exceed the old max-network-message # limit - assert(len(block.serialize(True)) > 2 * 1024 * 1024) + assert len(block.serialize(True)) > 2 * 1024 * 1024 test_witness_block(self.nodes[0], self.test_node, block, accepted=False) @@ -980,7 +978,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx[0].vout.pop() add_witness_commitment(block) block.solve() - assert(get_virtual_size(block) == MAX_BLOCK_BASE_SIZE) + assert get_virtual_size(block) == MAX_BLOCK_BASE_SIZE test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -998,14 +996,14 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block, nonce=1) block.vtx[0].wit = CTxWitness() # drop the nonce block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) - assert(self.nodes[0].getbestblockhash() != block.hash) + self.nodes[0].submitblock(block.serialize(True).hex()) + assert self.nodes[0].getbestblockhash() != block.hash # Now redo commitment with the standard nonce, but let bitcoind fill it in. add_witness_commitment(block, nonce=0) block.vtx[0].wit = CTxWitness() block.solve() - self.nodes[0].submitblock(bytes_to_hex_str(block.serialize(True))) + self.nodes[0].submitblock(block.serialize(True).hex()) assert_equal(self.nodes[0].getbestblockhash(), block.hash) # This time, add a tx with non-empty witness, but don't supply @@ -1020,9 +1018,9 @@ class SegWitTest(BitcoinTestFramework): block_2.vtx[0].vout.pop() block_2.vtx[0].wit = CTxWitness() - self.nodes[0].submitblock(bytes_to_hex_str(block_2.serialize(True))) + self.nodes[0].submitblock(block_2.serialize(True).hex()) # Tip should not advance! - assert(self.nodes[0].getbestblockhash() != block_2.hash) + assert self.nodes[0].getbestblockhash() != block_2.hash @subtest def test_extra_witness_data(self): @@ -1142,7 +1140,7 @@ class SegWitTest(BitcoinTestFramework): # This program is 19 max pushes (9937 bytes), then 64 more opcode-bytes. long_witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 63 + [OP_TRUE]) - assert(len(long_witness_program) == MAX_PROGRAM_LENGTH + 1) + assert len(long_witness_program) == MAX_PROGRAM_LENGTH + 1 long_witness_hash = sha256(long_witness_program) long_script_pubkey = CScript([OP_0, long_witness_hash]) @@ -1166,7 +1164,7 @@ class SegWitTest(BitcoinTestFramework): # Try again with one less byte in the witness program witness_program = CScript([b'a' * 520] * 19 + [OP_DROP] * 62 + [OP_TRUE]) - assert(len(witness_program) == MAX_PROGRAM_LENGTH) + assert len(witness_program) == MAX_PROGRAM_LENGTH witness_hash = sha256(witness_program) script_pubkey = CScript([OP_0, witness_hash]) @@ -1197,7 +1195,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(10): tx.vout.append(CTxOut(int(value / 10), script_pubkey)) tx.vout[0].nValue -= 1000 - assert(tx.vout[0].nValue >= 0) + assert tx.vout[0].nValue >= 0 block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) @@ -1347,8 +1345,8 @@ class SegWitTest(BitcoinTestFramework): assert_equal(raw_tx["vsize"], vsize) assert_equal(raw_tx["weight"], weight) assert_equal(len(raw_tx["vin"][0]["txinwitness"]), 1) - assert_equal(raw_tx["vin"][0]["txinwitness"][0], hexlify(witness_program).decode('ascii')) - assert(vsize != raw_tx["size"]) + assert_equal(raw_tx["vin"][0]["txinwitness"][0], witness_program.hex()) + assert vsize != raw_tx["size"] # Cleanup: mine the transactions and update utxo for next test self.nodes[0].generate(1) @@ -1398,7 +1396,7 @@ class SegWitTest(BitcoinTestFramework): self.nodes[0].generate(1) # Mine all the transactions sync_blocks(self.nodes) - assert(len(self.nodes[0].getrawmempool()) == 0) + assert len(self.nodes[0].getrawmempool()) == 0 # Finally, verify that version 0 -> version 1 transactions # are non-standard @@ -1667,7 +1665,7 @@ class SegWitTest(BitcoinTestFramework): # Create a slight bias for producing more utxos num_outputs = random.randint(1, 11) random.shuffle(temp_utxos) - assert(len(temp_utxos) > num_inputs) + assert len(temp_utxos) > num_inputs tx = CTransaction() total_value = 0 for i in range(num_inputs): @@ -1928,7 +1926,7 @@ class SegWitTest(BitcoinTestFramework): sync_blocks(self.nodes) # Make sure that this peer thinks segwit has activated. - assert(get_bip9_status(self.nodes[2], 'segwit')['status'] == "active") + assert get_bip9_status(self.nodes[2], 'segwit')['status'] == "active" # Make sure this peer's blocks match those of node0. height = self.nodes[2].getblockcount() @@ -1955,7 +1953,7 @@ class SegWitTest(BitcoinTestFramework): extra_sigops_available = MAX_SIGOP_COST % sigops_per_script # We chose the number of checkmultisigs/checksigs to make this work: - assert(extra_sigops_available < 100) # steer clear of MAX_OPS_PER_SCRIPT + assert extra_sigops_available < 100 # steer clear of MAX_OPS_PER_SCRIPT # This script, when spent with the first # N(=MAX_SIGOP_COST//sigops_per_script) outputs of our transaction, diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index c7ae57de86..7d7d251765 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -490,7 +490,7 @@ class SendHeadersTest(BitcoinTestFramework): # Now announce a header that forks the last two blocks tip = blocks[0].sha256 - height -= 1 + height -= 2 blocks = [] # Create extra blocks for later diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index 11299cbc00..027852809b 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. @@ -114,7 +114,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h1f.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_h1f.hash) # 4. Send another two block that build on the fork. @@ -131,7 +131,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h2f.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found # But this block should be accepted by node since it has equal work. self.nodes[0].getblock(block_h2f.hash) @@ -150,7 +150,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_h3.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found self.nodes[0].getblock(block_h3.hash) # But this block should be accepted by node since it has more work. @@ -263,7 +263,7 @@ class AcceptBlockTest(BitcoinTestFramework): if x['hash'] == block_292.hash: assert_equal(x['status'], "headers-only") tip_entry_found = True - assert(tip_entry_found) + assert tip_entry_found assert_raises_rpc_error(-1, "Block not found on disk", self.nodes[0].getblock, block_292.hash) test_node.send_message(msg_block(block_289f)) diff --git a/test/functional/rpc_bind.py b/test/functional/rpc_bind.py index 3938ca98dd..acc7cd811c 100755 --- a/test/functional/rpc_bind.py +++ b/test/functional/rpc_bind.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with the -rpcbind and -rpcallowip options.""" diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 31e60f1cea..facb05b54c 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPCs related to blockchainstate. @@ -35,6 +35,7 @@ from test_framework.util import ( from test_framework.blocktools import ( create_block, create_coinbase, + TIME_GENESIS_BLOCK, ) from test_framework.messages import ( msg_block, @@ -46,9 +47,11 @@ from test_framework.mininode import ( class BlockchainTest(BitcoinTestFramework): def set_test_params(self): + self.setup_clean_chain = True self.num_nodes = 1 def run_test(self): + self.mine_chain() self.restart_node(0, extra_args=['-stopatheight=207', '-prune=1']) # Set extra args with pruning after rescan is complete self._test_getblockchaininfo() @@ -61,6 +64,15 @@ class BlockchainTest(BitcoinTestFramework): self._test_waitforblockheight() assert self.nodes[0].verifychain(4, 0) + def mine_chain(self): + self.log.info('Create some old blocks') + address = self.nodes[0].get_deterministic_priv_key().address + for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): + # ten-minute steps from genesis block time + self.nodes[0].setmocktime(t) + self.nodes[0].generatetoaddress(1, address) + assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) + def _test_getblockchaininfo(self): self.log.info("Test getblockchaininfo") @@ -160,9 +172,9 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(chaintxstats['txcount'], 2) assert_equal(chaintxstats['window_final_block_hash'], b1_hash) assert_equal(chaintxstats['window_block_count'], 0) - assert('window_tx_count' not in chaintxstats) - assert('window_interval' not in chaintxstats) - assert('txrate' not in chaintxstats) + assert 'window_tx_count' not in chaintxstats + assert 'window_interval' not in chaintxstats + assert 'txrate' not in chaintxstats def _test_gettxoutsetinfo(self): node = self.nodes[0] diff --git a/test/functional/rpc_decodescript.py b/test/functional/rpc_decodescript.py index 940386eee7..01b8cb1854 100755 --- a/test/functional/rpc_decodescript.py +++ b/test/functional/rpc_decodescript.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test decoding scripts via decodescript RPC command.""" from test_framework.messages import CTransaction, sha256 from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, bytes_to_hex_str, hex_str_to_bytes +from test_framework.util import assert_equal, hex_str_to_bytes from io import BytesIO @@ -81,7 +81,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(multisig_script) assert_equal('2 ' + public_key + ' ' + public_key + ' ' + public_key + ' 3 OP_CHECKMULTISIG', rpc_result['asm']) # multisig in P2WSH - multisig_script_hash = bytes_to_hex_str(sha256(hex_str_to_bytes(multisig_script))) + multisig_script_hash = sha256(hex_str_to_bytes(multisig_script)).hex() assert_equal('0 ' + multisig_script_hash, rpc_result['segwit']['asm']) # 4) P2SH scriptPubKey @@ -119,7 +119,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(cltv_script) assert_equal('OP_IF ' + public_key + ' OP_CHECKSIGVERIFY OP_ELSE 500000 OP_CHECKLOCKTIMEVERIFY OP_DROP OP_ENDIF ' + public_key + ' OP_CHECKSIG', rpc_result['asm']) # CLTV script in P2WSH - cltv_script_hash = bytes_to_hex_str(sha256(hex_str_to_bytes(cltv_script))) + cltv_script_hash = sha256(hex_str_to_bytes(cltv_script)).hex() assert_equal('0 ' + cltv_script_hash, rpc_result['segwit']['asm']) # 7) P2PK scriptPubKey @@ -196,7 +196,7 @@ class DecodeScriptTest(BitcoinTestFramework): # some more full transaction tests of varying specific scriptSigs. used instead of # tests in decodescript_script_sig because the decodescript RPC is specifically # for working on scriptPubKeys (argh!). - push_signature = bytes_to_hex_str(txSave.vin[0].scriptSig)[2:(0x48*2+4)] + push_signature = txSave.vin[0].scriptSig.hex()[2:(0x48*2+4)] signature = push_signature[2:] der_signature = signature[:-2] signature_sighash_decoded = der_signature + '[ALL]' @@ -206,23 +206,23 @@ class DecodeScriptTest(BitcoinTestFramework): # 1) P2PK scriptSig txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # make sure that the sighash decodes come out correctly for a more complex / lesser used case. txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature_2) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 2) multisig scriptSig txSave.vin[0].scriptSig = hex_str_to_bytes('00' + push_signature + push_signature_2) - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('0 ' + signature_sighash_decoded + ' ' + signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 3) test a scriptSig that contains more than push operations. # in fact, it contains an OP_RETURN with data specially crafted to cause improper decode if the code does not catch it. txSave.vin[0].scriptSig = hex_str_to_bytes('6a143011020701010101010101020601010101010101') - rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) + rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('OP_RETURN 3011020701010101010101020601010101010101', rpc_result['vin'][0]['scriptSig']['asm']) def run_test(self): diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index 588bfbe083..9a21998d11 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -1,32 +1,29 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test deprecation of RPC calls.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_raises_rpc_error +# from test_framework.util import assert_raises_rpc_error class DeprecatedRpcTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True - self.extra_args = [[], ["-deprecatedrpc=generate"]] - - def skip_test_if_missing_module(self): - # The generate RPC method requires the wallet to be compiled - self.skip_if_no_wallet() + self.extra_args = [[], []] def run_test(self): # This test should be used to verify correct behaviour of deprecated # RPC methods with and without the -deprecatedrpc flags. For example: # - # self.log.info("Make sure that -deprecatedrpc=createmultisig allows it to take addresses") - # assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 1, [self.nodes[0].getnewaddress()]) - # self.nodes[1].createmultisig(1, [self.nodes[1].getnewaddress()]) - - self.log.info("Test generate RPC") - assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) - self.nodes[1].generate(1) + # In set_test_params: + # self.extra_args = [[], ["-deprecatedrpc=generate"]] + # + # In run_test: + # self.log.info("Test generate RPC") + # assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) + # self.nodes[1].generate(1) + self.log.info("No tested deprecated RPC methods") if __name__ == '__main__': DeprecatedRpcTest().main() diff --git a/test/functional/rpc_deriveaddresses.py b/test/functional/rpc_deriveaddresses.py new file mode 100755 index 0000000000..1984694692 --- /dev/null +++ b/test/functional/rpc_deriveaddresses.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the deriveaddresses rpc call.""" +from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import descsum_create +from test_framework.util import assert_equal, assert_raises_rpc_error + +class DeriveaddressesTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.supports_cli = 1 + + def run_test(self): + assert_raises_rpc_error(-5, "Invalid descriptor", self.nodes[0].deriveaddresses, "a") + + descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)#t6wfjs64" + address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" + assert_equal(self.nodes[0].deriveaddresses(descriptor), [address]) + + descriptor = descriptor[:-9] + assert_raises_rpc_error(-5, "Invalid descriptor", self.nodes[0].deriveaddresses, descriptor) + + descriptor_pubkey = "wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0)#s9ga3alw" + address = "bcrt1qjqmxmkpmxt80xz4y3746zgt0q3u3ferr34acd5" + assert_equal(self.nodes[0].deriveaddresses(descriptor_pubkey), [address]) + + ranged_descriptor = "wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)#kft60nuy" + assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, [1, 2]), ["bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"]) + assert_equal(self.nodes[0].deriveaddresses(ranged_descriptor, 2), [address, "bcrt1qhku5rq7jz8ulufe2y6fkcpnlvpsta7rq4442dy", "bcrt1qpgptk2gvshyl0s9lqshsmx932l9ccsv265tvaq"]) + + assert_raises_rpc_error(-8, "Range should not be specified for an un-ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"), [0, 2]) + + assert_raises_rpc_error(-8, "Range must be specified for a ranged descriptor", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)")) + + assert_raises_rpc_error(-8, "End of range is too high", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), 10000000000) + + assert_raises_rpc_error(-8, "Range is too large", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), [1000000000, 2000000000]) + + assert_raises_rpc_error(-8, "Range specified as [begin,end] must not have begin after end", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), [2, 0]) + + assert_raises_rpc_error(-8, "Range should be greater or equal than 0", self.nodes[0].deriveaddresses, descsum_create("wpkh(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)"), [-1, 0]) + + combo_descriptor = descsum_create("combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)") + assert_equal(self.nodes[0].deriveaddresses(combo_descriptor), ["mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", "mtfUoUax9L4tzXARpw1oTGxWyoogp52KhJ", address, "2NDvEwGfpEqJWfybzpKPHF2XH3jwoQV3D7x"]) + + hardened_without_privkey_descriptor = descsum_create("wpkh(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1'/1/0)") + assert_raises_rpc_error(-5, "Cannot derive script without private keys", self.nodes[0].deriveaddresses, hardened_without_privkey_descriptor) + + bare_multisig_descriptor = descsum_create("multi(1,tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/0,tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/1)") + assert_raises_rpc_error(-5, "Descriptor does not have a corresponding address", self.nodes[0].deriveaddresses, bare_multisig_descriptor) + +if __name__ == '__main__': + DeriveaddressesTest().main() diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 4f350953b2..d89fd6461f 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the fundrawtransaction RPC.""" @@ -94,7 +94,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) #test that we have enough inputs + assert len(dec_tx['vin']) > 0 #test that we have enough inputs ############################## # simple test with two coins # @@ -107,7 +107,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) #test if we have enough inputs + assert len(dec_tx['vin']) > 0 #test if we have enough inputs ############################## # simple test with two coins # @@ -120,7 +120,7 @@ class RawTransactionsTest(BitcoinTestFramework): rawtxfund = self.nodes[2].fundrawtransaction(rawtx) fee = rawtxfund['fee'] dec_tx = self.nodes[2].decoderawtransaction(rawtxfund['hex']) - assert(len(dec_tx['vin']) > 0) + assert len(dec_tx['vin']) > 0 assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') @@ -139,7 +139,7 @@ class RawTransactionsTest(BitcoinTestFramework): for out in dec_tx['vout']: totalOut += out['value'] - assert(len(dec_tx['vin']) > 0) + assert len(dec_tx['vin']) > 0 assert_equal(dec_tx['vin'][0]['scriptSig']['hex'], '') @@ -363,7 +363,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ ############################################################ @@ -378,7 +378,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -405,7 +405,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -438,7 +438,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance) + assert feeDelta >= 0 and feeDelta <= feeTolerance ############################################################ @@ -558,7 +558,7 @@ class RawTransactionsTest(BitcoinTestFramework): #compare fee feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) - assert(feeDelta >= 0 and feeDelta <= feeTolerance*19) #~19 inputs + assert feeDelta >= 0 and feeDelta <= feeTolerance*19 #~19 inputs ############################################# @@ -620,7 +620,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(len(res_dec["vin"]), 1) assert_equal(res_dec["vin"][0]["txid"], watchonly_txid) - assert("fee" in result.keys()) + assert "fee" in result.keys() assert_greater_than(result["changepos"], -1) ############################################################### @@ -635,16 +635,16 @@ class RawTransactionsTest(BitcoinTestFramework): result = self.nodes[3].fundrawtransaction(rawtx, True) res_dec = self.nodes[0].decoderawtransaction(result["hex"]) assert_equal(len(res_dec["vin"]), 2) - assert(res_dec["vin"][0]["txid"] == watchonly_txid or res_dec["vin"][1]["txid"] == watchonly_txid) + assert res_dec["vin"][0]["txid"] == watchonly_txid or res_dec["vin"][1]["txid"] == watchonly_txid assert_greater_than(result["fee"], 0) assert_greater_than(result["changepos"], -1) assert_equal(result["fee"] + res_dec["vout"][result["changepos"]]["value"], watchonly_amount / 10) signedtx = self.nodes[3].signrawtransactionwithwallet(result["hex"]) - assert(not signedtx["complete"]) + assert not signedtx["complete"] signedtx = self.nodes[0].signrawtransactionwithwallet(signedtx["hex"]) - assert(signedtx["complete"]) + assert signedtx["complete"] self.nodes[0].sendrawtransaction(signedtx["hex"]) self.nodes[0].generate(1) self.sync_all() @@ -676,10 +676,10 @@ class RawTransactionsTest(BitcoinTestFramework): for out in res_dec['vout']: if out['value'] > 1.0: changeaddress += out['scriptPubKey']['addresses'][0] - assert(changeaddress != "") + assert changeaddress != "" nextaddr = self.nodes[3].getnewaddress() # Now the change address key should be removed from the keypool - assert(changeaddress != nextaddr) + assert changeaddress != nextaddr ###################################### # Test subtractFeeFromOutputs option # diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index ca9e24367a..e17a8f6421 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -178,5 +178,10 @@ class GetblockstatsTest(BitcoinTestFramework): assert_raises_rpc_error(-5, 'Block not found', self.nodes[0].getblockstats, hash_or_height='000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f') + # Invalid number of args + assert_raises_rpc_error(-1, 'getblockstats hash_or_height ( stats )', self.nodes[0].getblockstats, '00', 1, 2) + assert_raises_rpc_error(-1, 'getblockstats hash_or_height ( stats )', self.nodes[0].getblockstats) + + if __name__ == '__main__': GetblockstatsTest().main() diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py new file mode 100755 index 0000000000..7bf8e68176 --- /dev/null +++ b/test/functional/rpc_misc.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test RPC misc output.""" +import xml.etree.ElementTree as ET + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_raises_rpc_error, + assert_equal, + assert_greater_than, + assert_greater_than_or_equal, +) + +from test_framework.authproxy import JSONRPCException + + +class RpcMiscTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + node = self.nodes[0] + + self.log.info("test getmemoryinfo") + memory = node.getmemoryinfo()['locked'] + assert_greater_than(memory['used'], 0) + assert_greater_than(memory['free'], 0) + assert_greater_than(memory['total'], 0) + # assert_greater_than_or_equal() for locked in case locking pages failed at some point + assert_greater_than_or_equal(memory['locked'], 0) + assert_greater_than(memory['chunks_used'], 0) + assert_greater_than(memory['chunks_free'], 0) + assert_equal(memory['used'] + memory['free'], memory['total']) + + self.log.info("test mallocinfo") + try: + mallocinfo = node.getmemoryinfo(mode="mallocinfo") + self.log.info('getmemoryinfo(mode="mallocinfo") call succeeded') + tree = ET.fromstring(mallocinfo) + assert_equal(tree.tag, 'malloc') + except JSONRPCException: + self.log.info('getmemoryinfo(mode="mallocinfo") not available') + assert_raises_rpc_error(-8, 'mallocinfo is only available when compiled with glibc 2.10+', node.getmemoryinfo, mode="mallocinfo") + + assert_raises_rpc_error(-8, "unknown mode foobar", node.getmemoryinfo, mode="foobar") + +if __name__ == '__main__': + RpcMiscTest().main() diff --git a/test/functional/rpc_named_arguments.py b/test/functional/rpc_named_arguments.py index 6c5c29dfdf..ecac9c2f82 100755 --- a/test/functional/rpc_named_arguments.py +++ b/test/functional/rpc_named_arguments.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test using named arguments for RPCs.""" @@ -17,7 +17,7 @@ class NamedArgumentTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] h = node.help(command='getblockchaininfo') - assert(h.startswith('getblockchaininfo\n')) + assert h.startswith('getblockchaininfo\n') assert_raises_rpc_error(-8, 'Unknown named parameter', node.help, random='getblockchaininfo') diff --git a/test/functional/rpc_preciousblock.py b/test/functional/rpc_preciousblock.py index 72e6e6329f..da228df9c3 100755 --- a/test/functional/rpc_preciousblock.py +++ b/test/functional/rpc_preciousblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the preciousblock RPC.""" @@ -16,7 +16,7 @@ def unidirectional_node_sync_via_rpc(node_src, node_dest): blockhash = node_src.getbestblockhash() while True: try: - assert(len(node_dest.getblock(blockhash, False)) > 0) + assert len(node_dest.getblock(blockhash, False)) > 0 break except: blocks_to_copy.append(blockhash) @@ -24,7 +24,7 @@ def unidirectional_node_sync_via_rpc(node_src, node_dest): blocks_to_copy.reverse() for blockhash in blocks_to_copy: blockdata = node_src.getblock(blockhash, False) - assert(node_dest.submitblock(blockdata) in (None, 'inconclusive')) + assert node_dest.submitblock(blockdata) in (None, 'inconclusive') def node_sync_via_rpc(nodes): for node_src in nodes: @@ -57,7 +57,7 @@ class PreciousTest(BitcoinTestFramework): self.log.info("Mine competing blocks E-F-G on Node 1") hashG = self.nodes[1].generatetoaddress(3, gen_address(1))[-1] assert_equal(self.nodes[1].getblockcount(), 5) - assert(hashC != hashG) + assert hashC != hashG self.log.info("Connect nodes and check no reorg occurs") # Submit competing blocks via RPC so any reorg should occur before we proceed (no way to wait on inaction for p2p sync) node_sync_via_rpc(self.nodes[0:2]) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 1e10280e60..885e07c4dd 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the Partially Signed Transaction RPCs. """ +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, find_output, disconnect_nodes, connect_nodes_bi, sync_blocks +from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes_bi, disconnect_nodes, find_output, sync_blocks import json import os @@ -19,8 +20,6 @@ class PSBTTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = False self.num_nodes = 3 - # TODO: remove -txindex. Currently required for getrawtransaction call. - self.extra_args = [[], ["-txindex"], ["-txindex"]] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -48,11 +47,11 @@ class PSBTTest(BitcoinTestFramework): utxos = online_node.listunspent(addresses=[offline_addr]) raw = online_node.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}]) psbt = online_node.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"] - assert("non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0]) + assert "non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0] # Have the offline node sign the PSBT (which will update the UTXO to segwit) signed_psbt = offline_node.walletprocesspsbt(psbt)["psbt"] - assert("witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0]) + assert "witness_utxo" in mining_node.decodepsbt(signed_psbt)["inputs"][0] # Make sure we can mine the resulting transaction txid = mining_node.sendrawtransaction(mining_node.finalizepsbt(signed_psbt)["hex"]) @@ -160,11 +159,11 @@ class PSBTTest(BitcoinTestFramework): node1_addr = self.nodes[1].getnewaddress() node2_addr = self.nodes[2].getnewaddress() txid1 = self.nodes[0].sendtoaddress(node1_addr, 13) - txid2 =self.nodes[0].sendtoaddress(node2_addr, 13) - self.nodes[0].generate(6) + txid2 = self.nodes[0].sendtoaddress(node2_addr, 13) + blockhash = self.nodes[0].generate(6)[0] self.sync_all() - vout1 = find_output(self.nodes[1], txid1, 13) - vout2 = find_output(self.nodes[2], txid2, 13) + vout1 = find_output(self.nodes[1], txid1, 13, blockhash=blockhash) + vout2 = find_output(self.nodes[2], txid2, 13, blockhash=blockhash) # Create a psbt spending outputs from nodes 1 and 2 psbt_orig = self.nodes[0].createpsbt([{"txid":txid1, "vout":vout1}, {"txid":txid2, "vout":vout2}], {self.nodes[0].getnewaddress():25.999}) @@ -269,6 +268,9 @@ class PSBTTest(BitcoinTestFramework): combined = self.nodes[2].combinepsbt(combiner['combine']) assert_equal(combined, combiner['result']) + # Empty combiner test + assert_raises_rpc_error(-8, "Parameter 'txs' cannot be empty", self.nodes[0].combinepsbt, []) + # Finalizer test for finalizer in finalizers: finalized = self.nodes[2].finalizepsbt(finalizer['finalize'], False)['psbt'] @@ -290,5 +292,75 @@ class PSBTTest(BitcoinTestFramework): psbt = self.nodes[1].walletcreatefundedpsbt([], [{p2pkh : 1}], 0, {"includeWatching" : True}, True) self.nodes[0].decodepsbt(psbt['psbt']) + # Test decoding error: invalid base64 + assert_raises_rpc_error(-22, "TX decode failed invalid base64", self.nodes[0].decodepsbt, ";definitely not base64;") + + # Send to all types of addresses + addr1 = self.nodes[1].getnewaddress("", "bech32") + txid1 = self.nodes[0].sendtoaddress(addr1, 11) + vout1 = find_output(self.nodes[0], txid1, 11) + addr2 = self.nodes[1].getnewaddress("", "legacy") + txid2 = self.nodes[0].sendtoaddress(addr2, 11) + vout2 = find_output(self.nodes[0], txid2, 11) + addr3 = self.nodes[1].getnewaddress("", "p2sh-segwit") + txid3 = self.nodes[0].sendtoaddress(addr3, 11) + vout3 = find_output(self.nodes[0], txid3, 11) + self.sync_all() + + # Update a PSBT with UTXOs from the node + # Bech32 inputs should be filled with witness UTXO. Other inputs should not be filled because they are non-witness + psbt = self.nodes[1].createpsbt([{"txid":txid1, "vout":vout1},{"txid":txid2, "vout":vout2},{"txid":txid3, "vout":vout3}], {self.nodes[0].getnewaddress():32.999}) + decoded = self.nodes[1].decodepsbt(psbt) + assert "witness_utxo" not in decoded['inputs'][0] and "non_witness_utxo" not in decoded['inputs'][0] + assert "witness_utxo" not in decoded['inputs'][1] and "non_witness_utxo" not in decoded['inputs'][1] + assert "witness_utxo" not in decoded['inputs'][2] and "non_witness_utxo" not in decoded['inputs'][2] + updated = self.nodes[1].utxoupdatepsbt(psbt) + decoded = self.nodes[1].decodepsbt(updated) + assert "witness_utxo" in decoded['inputs'][0] and "non_witness_utxo" not in decoded['inputs'][0] + assert "witness_utxo" not in decoded['inputs'][1] and "non_witness_utxo" not in decoded['inputs'][1] + assert "witness_utxo" not in decoded['inputs'][2] and "non_witness_utxo" not in decoded['inputs'][2] + + # Two PSBTs with a common input should not be joinable + psbt1 = self.nodes[1].createpsbt([{"txid":txid1, "vout":vout1}], {self.nodes[0].getnewaddress():Decimal('10.999')}) + assert_raises_rpc_error(-8, "exists in multiple PSBTs", self.nodes[1].joinpsbts, [psbt1, updated]) + + # Join two distinct PSBTs + addr4 = self.nodes[1].getnewaddress("", "p2sh-segwit") + txid4 = self.nodes[0].sendtoaddress(addr4, 5) + vout4 = find_output(self.nodes[0], txid4, 5) + self.nodes[0].generate(6) + self.sync_all() + psbt2 = self.nodes[1].createpsbt([{"txid":txid4, "vout":vout4}], {self.nodes[0].getnewaddress():Decimal('4.999')}) + psbt2 = self.nodes[1].walletprocesspsbt(psbt2)['psbt'] + psbt2_decoded = self.nodes[0].decodepsbt(psbt2) + assert "final_scriptwitness" in psbt2_decoded['inputs'][0] and "final_scriptSig" in psbt2_decoded['inputs'][0] + joined = self.nodes[0].joinpsbts([psbt, psbt2]) + joined_decoded = self.nodes[0].decodepsbt(joined) + assert len(joined_decoded['inputs']) == 4 and len(joined_decoded['outputs']) == 2 and "final_scriptwitness" not in joined_decoded['inputs'][3] and "final_scriptSig" not in joined_decoded['inputs'][3] + + # Newly created PSBT needs UTXOs and updating + addr = self.nodes[1].getnewaddress("", "p2sh-segwit") + txid = self.nodes[0].sendtoaddress(addr, 7) + addrinfo = self.nodes[1].getaddressinfo(addr) + blockhash = self.nodes[0].generate(6)[0] + self.sync_all() + vout = find_output(self.nodes[0], txid, 7, blockhash=blockhash) + psbt = self.nodes[1].createpsbt([{"txid":txid, "vout":vout}], {self.nodes[0].getnewaddress("", "p2sh-segwit"):Decimal('6.999')}) + analyzed = self.nodes[0].analyzepsbt(psbt) + assert not analyzed['inputs'][0]['has_utxo'] and not analyzed['inputs'][0]['is_final'] and analyzed['inputs'][0]['next'] == 'updater' and analyzed['next'] == 'updater' + + # After update with wallet, only needs signing + updated = self.nodes[1].walletprocesspsbt(psbt, False, 'ALL', True)['psbt'] + analyzed = self.nodes[0].analyzepsbt(updated) + assert analyzed['inputs'][0]['has_utxo'] and not analyzed['inputs'][0]['is_final'] and analyzed['inputs'][0]['next'] == 'signer' and analyzed['next'] == 'signer' and analyzed['inputs'][0]['missing']['signatures'][0] == addrinfo['embedded']['witness_program'] + + # Check fee and size things + assert analyzed['fee'] == Decimal('0.001') and analyzed['estimated_vsize'] == 134 and analyzed['estimated_feerate'] == '0.00746268 BTC/kB' + + # After signing and finalizing, needs extracting + signed = self.nodes[1].walletprocesspsbt(updated)['psbt'] + analyzed = self.nodes[0].analyzepsbt(signed) + assert analyzed['inputs'][0]['has_utxo'] and analyzed['inputs'][0]['is_final'] and analyzed['next'] == 'extractor' + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index a97d753626..fba9e75268 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the rawtransaction RPCs. @@ -17,7 +17,7 @@ from decimal import Decimal from io import BytesIO from test_framework.messages import CTransaction, ToHex from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes_bi, hex_str_to_bytes +from test_framework.util import assert_equal, assert_raises_rpc_error, connect_nodes_bi, hex_str_to_bytes class multidict(dict): """Dictionary that allows duplicate keys. @@ -42,7 +42,6 @@ class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 3 - # TODO: remove -txindex. Currently required for getrawtransaction call. self.extra_args = [["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"], ["-addresstype=legacy", "-txindex"]] def skip_test_if_missing_module(self): @@ -120,21 +119,21 @@ class RawTransactionsTest(BitcoinTestFramework): tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs={address: 99})))) assert_equal(len(tx.vout), 1) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}]), ) # Two outputs tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=OrderedDict([(address, 99), (address2, 99)]))))) assert_equal(len(tx.vout), 2) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}]), ) # Multiple mixed outputs tx.deserialize(BytesIO(hex_str_to_bytes(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([(address, 99), (address2, 99), ('data', '99')]))))) assert_equal(len(tx.vout), 3) assert_equal( - bytes_to_hex_str(tx.serialize()), + tx.serialize().hex(), self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}, {'data': '99'}]), ) diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index 11b4db6ec5..6346477922 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the scantxoutset rpc call.""" @@ -95,11 +95,12 @@ class ScantxoutsetTest(BitcoinTestFramework): assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/*)", "range": 1500}])['total_amount'], Decimal("28.672")) assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1499}])['total_amount'], Decimal("12.288")) assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])['total_amount'], Decimal("28.672")) + assert_equal(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": [1500,1500]}])['total_amount'], Decimal("16.384")) # Test the reported descriptors for a few matches - assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0'/*)", "range": 1499}])), ["pkh([0c5f9a1e/0'/0'/0]026dbd8b2315f296d36e6b6920b1579ca75569464875c7ebe869b536a7d9503c8c)", "pkh([0c5f9a1e/0'/0'/1]033e6f25d76c00bedb3a8993c7d5739ee806397f0529b1b31dda31ef890f19a60c)"]) - assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"])), ["pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)"]) - assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])), ['pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)', 'pkh([0c5f9a1e/1/1/1500]03832901c250025da2aebae2bfb38d5c703a57ab66ad477f9c578bfbcd78abca6f)', 'pkh([0c5f9a1e/1/1/1]030d820fc9e8211c4169be8530efbc632775d8286167afd178caaf1089b77daba7)']) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/0h/0'/*)", "range": 1499}])), ["pkh([0c5f9a1e/0'/0'/0]026dbd8b2315f296d36e6b6920b1579ca75569464875c7ebe869b536a7d9503c8c)#dzxw429x", "pkh([0c5f9a1e/0'/0'/1]033e6f25d76c00bedb3a8993c7d5739ee806397f0529b1b31dda31ef890f19a60c)#43rvceed"]) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ "combo(tprv8ZgxMBicQKsPd7Uf69XL1XwhmjHopUGep8GuEiJDZmbQz6o58LninorQAfcKZWARbtRtfnLcJ5MQ2AtHcQJCCRUcMRvmDUjyEmNUWwx8UbK/1/1/0)"])), ["pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8"]) + assert_equal(descriptors(self.nodes[0].scantxoutset("start", [ {"desc": "combo(tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B/1/1/*)", "range": 1500}])), ['pkh([0c5f9a1e/1/1/0]03e1c5b6e650966971d7e71ef2674f80222752740fc1dfd63bbbd220d2da9bd0fb)#cxmct4w8', 'pkh([0c5f9a1e/1/1/1500]03832901c250025da2aebae2bfb38d5c703a57ab66ad477f9c578bfbcd78abca6f)#vchwd07g', 'pkh([0c5f9a1e/1/1/1]030d820fc9e8211c4169be8530efbc632775d8286167afd178caaf1089b77daba7)#z2t3ypsa']) if __name__ == '__main__': ScantxoutsetTest().main() diff --git a/test/functional/rpc_signmessage.py b/test/functional/rpc_signmessage.py index ad0e29b451..0cb3ce4215 100755 --- a/test/functional/rpc_signmessage.py +++ b/test/functional/rpc_signmessage.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC commands for signing and verifying messages.""" @@ -25,18 +25,18 @@ class SignMessagesTest(BitcoinTestFramework): expected_signature = 'INbVnW4e6PeRmsv2Qgu8NuopvrVjkcxob+sX8OcZG0SALhWybUjzMLPdAsXI46YZGb0KQTRii+wWIQzRpG/U+S0=' signature = self.nodes[0].signmessagewithprivkey(priv_key, message) assert_equal(expected_signature, signature) - assert(self.nodes[0].verifymessage(address, signature, message)) + assert self.nodes[0].verifymessage(address, signature, message) self.log.info('test signing with an address with wallet') address = self.nodes[0].getnewaddress() signature = self.nodes[0].signmessage(address, message) - assert(self.nodes[0].verifymessage(address, signature, message)) + assert self.nodes[0].verifymessage(address, signature, message) self.log.info('test verifying with another address should not work') other_address = self.nodes[0].getnewaddress() other_signature = self.nodes[0].signmessage(other_address, message) - assert(not self.nodes[0].verifymessage(other_address, signature, message)) - assert(not self.nodes[0].verifymessage(address, other_signature, message)) + assert not self.nodes[0].verifymessage(other_address, signature, message) + assert not self.nodes[0].verifymessage(address, other_signature, message) if __name__ == '__main__': SignMessagesTest().main() diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index 291538df64..dcafc82857 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -1,18 +1,21 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction signing using the signrawtransaction* RPCs.""" from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.util import assert_equal, assert_raises_rpc_error, hex_str_to_bytes +from test_framework.messages import sha256 +from test_framework.script import CScript, OP_0 +from decimal import Decimal class SignRawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 1 - self.extra_args = [["-deprecatedrpc=signrawtransaction"]] + self.num_nodes = 2 + self.extra_args = [["-deprecatedrpc=signrawtransaction"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -143,9 +146,33 @@ class SignRawTransactionsTest(BitcoinTestFramework): assert_equal(rawTxSigned['errors'][1]['witness'], ["304402203609e17b84f6a7d30c80bfa610b5b4542f32a8a0d5447a12fb1366d7f01cc44a0220573a954c4518331561406f90300e8f3358f51928d43c212a8caed02de67eebee01", "025476c2e83188368da1ff3e292e7acafcdb3566bb0ad253f62fc70f07aeee6357"]) assert not rawTxSigned['errors'][0]['witness'] + def witness_script_test(self): + # Now test signing transaction to P2SH-P2WSH addresses without wallet + # Create a new P2SH-P2WSH 1-of-1 multisig address: + embedded_address = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress()) + embedded_privkey = self.nodes[1].dumpprivkey(embedded_address["address"]) + p2sh_p2wsh_address = self.nodes[1].addmultisigaddress(1, [embedded_address["pubkey"]], "", "p2sh-segwit") + # send transaction to P2SH-P2WSH 1-of-1 multisig address + self.nodes[0].generate(101) + self.nodes[0].sendtoaddress(p2sh_p2wsh_address["address"], 49.999) + self.nodes[0].generate(1) + self.sync_all() + # Find the UTXO for the transaction node[1] should have received, check witnessScript matches + unspent_output = self.nodes[1].listunspent(0, 999999, [p2sh_p2wsh_address["address"]])[0] + assert_equal(unspent_output["witnessScript"], p2sh_p2wsh_address["redeemScript"]) + p2sh_redeemScript = CScript([OP_0, sha256(hex_str_to_bytes(p2sh_p2wsh_address["redeemScript"]))]) + assert_equal(unspent_output["redeemScript"], p2sh_redeemScript.hex()) + # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys + spending_tx = self.nodes[0].createrawtransaction([unspent_output], {self.nodes[1].getnewaddress(): Decimal("49.998")}) + spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [unspent_output]) + # Check the signing completed successfully + assert 'complete' in spending_tx_signed + assert_equal(spending_tx_signed['complete'], True) + def run_test(self): self.successful_signing_test() self.script_verification_error_test() + self.witness_script_test() self.test_with_lock_outputs() diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index 20b6341550..e86f91b1d0 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC call related to the uptime command. @@ -23,7 +23,7 @@ class UptimeTest(BitcoinTestFramework): def _test_uptime(self): wait_time = 10 self.nodes[0].setmocktime(int(time.time() + wait_time)) - assert(self.nodes[0].uptime() >= wait_time) + assert self.nodes[0].uptime() >= wait_time if __name__ == '__main__': diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 456d43aa2e..f36cffe957 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Encode and decode BASE58, P2PKH and P2SH addresses.""" from .script import hash256, hash160, sha256, CScript, OP_0 -from .util import bytes_to_hex_str, hex_str_to_bytes +from .util import hex_str_to_bytes from . import segwit_addr @@ -16,9 +16,9 @@ chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' def byte_to_base58(b, version): result = '' - str = bytes_to_hex_str(b) - str = bytes_to_hex_str(chr(version).encode('latin-1')) + str - checksum = bytes_to_hex_str(hash256(hex_str_to_bytes(str))) + str = b.hex() + str = chr(version).encode('latin-1').hex() + str + checksum = hash256(hex_str_to_bytes(str)).hex() str += checksum[:8] value = int('0x'+str,0) while value > 0: @@ -32,12 +32,12 @@ def byte_to_base58(b, version): # TODO: def base58_decode def keyhash_to_p2pkh(hash, main = False): - assert (len(hash) == 20) + assert len(hash) == 20 version = 0 if main else 111 return byte_to_base58(hash, version) def scripthash_to_p2sh(hash, main = False): - assert (len(hash) == 20) + assert len(hash) == 20 version = 5 if main else 196 return byte_to_base58(hash, version) @@ -80,11 +80,11 @@ def check_key(key): key = hex_str_to_bytes(key) # Assuming this is hex string if (type(key) is bytes and (len(key) == 33 or len(key) == 65)): return key - assert(False) + assert False def check_script(script): if (type(script) is str): script = hex_str_to_bytes(script) # Assuming this is hex string if (type(script) is bytes or type(script) is CScript): return script - assert(False) + assert False diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 6b47cae4c3..7ac044d0d0 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" @@ -20,7 +20,6 @@ from .messages import ( CTxOut, FromHex, ToHex, - bytes_to_hex_str, hash256, hex_str_to_bytes, ser_string, @@ -43,9 +42,13 @@ from io import BytesIO MAX_BLOCK_SIGOPS = 20000 +# Genesis block time (regtest) +TIME_GENESIS_BLOCK = 1296688602 + # From BIP141 WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed" + def create_block(hashprev, coinbase, ntime=None, *, version=1): """Create a block (with regtest difficulty).""" block = CBlock() @@ -128,7 +131,7 @@ def create_tx_with_script(prevtx, n, script_sig=b"", *, amount, script_pub_key=C Can optionally pass scriptPubKey and scriptSig, default is anyone-can-spend output. """ tx = CTransaction() - assert(n < len(prevtx.vout)) + assert n < len(prevtx.vout) tx.vin.append(CTxIn(COutPoint(prevtx.sha256, n), script_sig, 0xffffffff)) tx.vout.append(CTxOut(amount, script_pub_key)) tx.calc_sha256() @@ -186,7 +189,7 @@ def witness_script(use_p2wsh, pubkey): witness_program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) scripthash = sha256(witness_program) pkscript = CScript([OP_0, scripthash]) - return bytes_to_hex_str(pkscript) + return pkscript.hex() def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): """Return a transaction (in hex) that spends the given utxo to a segwit output. @@ -211,7 +214,7 @@ def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=Tru tx_to_witness = create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount) if (sign): signed = node.signrawtransactionwithwallet(tx_to_witness) - assert("errors" not in signed or len(["errors"]) == 0) + assert "errors" not in signed or len(["errors"]) == 0 return node.sendrawtransaction(signed["hex"]) else: if (insert_redeem_script): diff --git a/test/functional/test_framework/descriptors.py b/test/functional/test_framework/descriptors.py new file mode 100644 index 0000000000..29482ce01e --- /dev/null +++ b/test/functional/test_framework/descriptors.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Utility functions related to output descriptors""" + +INPUT_CHARSET = "0123456789()[],'/*abcdefgh@:$%{}IJKLMNOPQRSTUVWXYZ&+-.;<=>?!^_|~ijklmnopqrstuvwxyzABCDEFGH`#\"\\ " +CHECKSUM_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" +GENERATOR = [0xf5dee51989, 0xa9fdca3312, 0x1bab10e32d, 0x3706b1677a, 0x644d626ffd] + +def descsum_polymod(symbols): + """Internal function that computes the descriptor checksum.""" + chk = 1 + for value in symbols: + top = chk >> 35 + chk = (chk & 0x7ffffffff) << 5 ^ value + for i in range(5): + chk ^= GENERATOR[i] if ((top >> i) & 1) else 0 + return chk + +def descsum_expand(s): + """Internal function that does the character to symbol expansion""" + groups = [] + symbols = [] + for c in s: + if not c in INPUT_CHARSET: + return None + v = INPUT_CHARSET.find(c) + symbols.append(v & 31) + groups.append(v >> 5) + if len(groups) == 3: + symbols.append(groups[0] * 9 + groups[1] * 3 + groups[2]) + groups = [] + if len(groups) == 1: + symbols.append(groups[0]) + elif len(groups) == 2: + symbols.append(groups[0] * 3 + groups[1]) + return symbols + +def descsum_create(s): + """Add a checksum to a descriptor without""" + symbols = descsum_expand(s) + [0, 0, 0, 0, 0, 0, 0, 0] + checksum = descsum_polymod(symbols) ^ 1 + return s + '#' + ''.join(CHECKSUM_CHARSET[(checksum >> (5 * (7 - i))) & 31] for i in range(8)) + +def descsum_check(s, require=True): + """Verify that the checksum is correct in a descriptor""" + if not '#' in s: + return not require + if s[-9] != '#': + return False + if not all(x in CHECKSUM_CHARSET for x in s[-8:]): + return False + symbols = descsum_expand(s[:-9]) + [CHECKSUM_CHARSET.find(x) for x in s[-8:]] + return descsum_polymod(symbols) == 1 diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 356a45d6d0..7cf51d9223 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2018 The Bitcoin Core developers +# Copyright (c) 2010-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin test framework primitive and message structures @@ -28,7 +28,7 @@ import struct import time from test_framework.siphash import siphash256 -from test_framework.util import hex_str_to_bytes, bytes_to_hex_str +from test_framework.util import hex_str_to_bytes, assert_equal MIN_VERSION_SUPPORTED = 60001 MY_VERSION = 70014 # past bip-31 for ping/pong @@ -181,7 +181,7 @@ def FromHex(obj, hex_string): # Convert a binary-serializable object to hex (eg for submission via RPC) def ToHex(obj): - return bytes_to_hex_str(obj.serialize()) + return obj.serialize().hex() # Objects that map to bitcoind objects, which can be serialized/deserialized @@ -319,7 +319,7 @@ class CTxIn: def __repr__(self): return "CTxIn(prevout=%s scriptSig=%s nSequence=%i)" \ - % (repr(self.prevout), bytes_to_hex_str(self.scriptSig), + % (repr(self.prevout), self.scriptSig.hex(), self.nSequence) @@ -343,7 +343,7 @@ class CTxOut: def __repr__(self): return "CTxOut(nValue=%i.%08i scriptPubKey=%s)" \ % (self.nValue // COIN, self.nValue % COIN, - bytes_to_hex_str(self.scriptPubKey)) + self.scriptPubKey.hex()) class CScriptWitness: @@ -355,7 +355,7 @@ class CScriptWitness: def __repr__(self): return "CScriptWitness(%s)" % \ - (",".join([bytes_to_hex_str(x) for x in self.stack])) + (",".join([x.hex() for x in self.stack])) def is_null(self): if self.stack: @@ -591,6 +591,8 @@ class CBlockHeader: % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, time.ctime(self.nTime), self.nBits, self.nNonce) +BLOCK_HEADER_SIZE = len(CBlockHeader().serialize()) +assert_equal(BLOCK_HEADER_SIZE, 80) class CBlock(CBlockHeader): __slots__ = ("vtx",) diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index ac7cc068bd..52a840941b 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2018 The Bitcoin Core developers +# Copyright (c) 2010-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin P2P network half-a-node. @@ -218,10 +218,7 @@ class P2PConnection(asyncio.Protocol): def maybe_write(): if not self._transport: return - # Python <3.4.4 does not have is_closing, so we have to check for - # its existence explicitly as long as Bitcoin Core supports all - # Python 3.4 versions. - if hasattr(self._transport, 'is_closing') and self._transport.is_closing(): + if self._transport.is_closing(): return self._transport.write(raw_message_bytes) NetworkThread.network_event_loop.call_soon_threadsafe(maybe_write) diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index 45c182f630..c98424e8e2 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Linux network utilities. @@ -12,7 +12,7 @@ import socket import struct import array import os -from binascii import unhexlify, hexlify +from binascii import unhexlify # STATE_ESTABLISHED = '01' # STATE_SYN_SENT = '02' @@ -129,17 +129,17 @@ def addr_to_hex(addr): if i == 0 or i == (len(addr)-1): # skip empty component at beginning or end continue x += 1 # :: skips to suffix - assert(x < 2) + assert x < 2 else: # two bytes per component val = int(comp, 16) sub[x].append(val >> 8) sub[x].append(val & 0xff) nullbytes = 16 - len(sub[0]) - len(sub[1]) - assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) + assert (x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0) addr = sub[0] + ([0] * nullbytes) + sub[1] else: raise ValueError('Could not parse address %s' % addr) - return hexlify(bytearray(addr)).decode('ascii') + return bytearray(addr).hex() def test_ipv6_local(): ''' diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 012c80a1be..384062b808 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Functionality to build scripts, as well as SignatureHash(). @@ -9,7 +9,6 @@ This file is modified from python-bitcoinlib. from .messages import CTransaction, CTxOut, sha256, hash256, uint256_from_str, ser_uint256, ser_string -from binascii import hexlify import hashlib import struct @@ -450,10 +449,6 @@ class CScript(bytes): # join makes no sense for a CScript() raise NotImplementedError - # Python 3.4 compatibility - def hex(self): - return hexlify(self).decode('ascii') - def __new__(cls, value=b''): if isinstance(value, bytes) or isinstance(value, bytearray): return super(CScript, cls).__new__(cls, value) @@ -545,7 +540,7 @@ class CScript(bytes): def __repr__(self): def _repr(o): if isinstance(o, bytes): - return "x('%s')" % hexlify(o).decode('ascii') + return "x('%s')" % o.hex() else: return repr(o) diff --git a/test/functional/test_framework/socks5.py b/test/functional/test_framework/socks5.py index a21c864e75..799b1c74b8 100644 --- a/test/functional/test_framework/socks5.py +++ b/test/functional/test_framework/socks5.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Dummy Socks5 server for testing.""" @@ -144,7 +144,7 @@ class Socks5Server(): thread.start() def start(self): - assert(not self.running) + assert not self.running self.running = True self.thread = threading.Thread(None, self.run) self.thread.daemon = True diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 8c4c0d7226..15d2e08c93 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" @@ -29,11 +29,11 @@ from .util import ( get_datadir_path, initialize_datadir, p2p_port, - set_node_times, sync_blocks, sync_mempools, ) + class TestStatus(Enum): PASSED = 1 FAILED = 2 @@ -94,7 +94,6 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.setup_clean_chain = False self.nodes = [] self.network_thread = None - self.mocktime = 0 self.rpc_timeout = 60 # Wait for up to 60 seconds for the RPC server to respond self.supports_cli = False self.bind_to_localhost_only = True @@ -275,6 +274,19 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.add_nodes(self.num_nodes, extra_args) self.start_nodes() self.import_deterministic_coinbase_privkeys() + if not self.setup_clean_chain: + for n in self.nodes: + assert_equal(n.getblockchaininfo()["blocks"], 199) + # To ensure that all nodes are out of IBD, the most recent block + # must have a timestamp not too old (see IsInitialBlockDownload()). + self.log.debug('Generate a block with current time') + block_hash = self.nodes[0].generate(1)[0] + block = self.nodes[0].getblock(blockhash=block_hash, verbosity=0) + for n in self.nodes: + n.submitblock(block) + chain_info = n.getblockchaininfo() + assert_equal(chain_info["blocks"], 200) + assert_equal(chain_info["initialblockdownload"], False) def import_deterministic_coinbase_privkeys(self): for n in self.nodes: @@ -316,8 +328,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): timewait=self.rpc_timeout, bitcoind=binary[i], bitcoin_cli=self.options.bitcoincli, - mocktime=self.mocktime, coverage_dir=self.options.coveragedir, + cwd=self.options.tmpdir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli, @@ -434,7 +446,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def _initialize_chain(self): """Initialize a pre-mined blockchain for use by the test. - Create a cache of a 200-block-long chain (with wallet) for MAX_NODES + Create a cache of a 199-block-long chain (with wallet) for MAX_NODES Afterward, create num_nodes copies from the cache.""" assert self.num_nodes <= MAX_NODES @@ -467,8 +479,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): timewait=self.rpc_timeout, bitcoind=self.options.bitcoind, bitcoin_cli=self.options.bitcoincli, - mocktime=self.mocktime, coverage_dir=None, + cwd=self.options.tmpdir, )) self.nodes[i].args = args self.start_node(i) @@ -477,32 +489,22 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): for node in self.nodes: node.wait_for_rpc_connection() - # For backward compatibility of the python scripts with previous - # versions of the cache, set mocktime to Jan 1, - # 2014 + (201 * 10 * 60)""" - self.mocktime = 1388534400 + (201 * 10 * 60) - - # Create a 200-block-long chain; each of the 4 first nodes + # Create a 199-block-long chain; each of the 4 first nodes # gets 25 mature blocks and 25 immature. - # Note: To preserve compatibility with older versions of - # initialize_chain, only 4 nodes will generate coins. - # - # blocks are created with timestamps 10 minutes apart - # starting from 2010 minutes in the past - block_time = self.mocktime - (201 * 10 * 60) - for i in range(2): - for peer in range(4): - for j in range(25): - set_node_times(self.nodes, block_time) - self.nodes[peer].generatetoaddress(1, self.nodes[peer].get_deterministic_priv_key().address) - block_time += 10 * 60 - # Must sync before next peer starts generating blocks - sync_blocks(self.nodes) + # The 4th node gets only 24 immature blocks so that the very last + # block in the cache does not age too much (have an old tip age). + # This is needed so that we are out of IBD when the test starts, + # see the tip age check in IsInitialBlockDownload(). + for i in range(8): + self.nodes[0].generatetoaddress(25 if i != 7 else 24, self.nodes[i % 4].get_deterministic_priv_key().address) + sync_blocks(self.nodes) + + for n in self.nodes: + assert_equal(n.getblockchaininfo()["blocks"], 199) # Shut them down, and clean up cache directories: self.stop_nodes() self.nodes = [] - self.mocktime = 0 def cache_path(n, *paths): return os.path.join(get_datadir_path(self.options.cachedir, n), "regtest", *paths) diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index d6e31457d1..ec5d3b267e 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Class for bitcoind node under test""" @@ -31,9 +31,6 @@ from .util import ( p2p_port, ) -# For Python 3.4 compatibility -JSONDecodeError = getattr(json, "JSONDecodeError", ValueError) - BITCOIND_PROC_WAIT_TIMEOUT = 60 @@ -61,7 +58,7 @@ class TestNode(): To make things easier for the test writer, any unrecognised messages will be dispatched to the RPC connection.""" - def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, mocktime, coverage_dir, extra_conf=None, extra_args=None, use_cli=False, start_perf=False): + def __init__(self, i, datadir, *, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False): """ Kwargs: start_perf (bool): If True, begin profiling the node with `perf` as soon as @@ -76,6 +73,7 @@ class TestNode(): self.rpc_timeout = timewait self.binary = bitcoind self.coverage_dir = coverage_dir + self.cwd = cwd if extra_conf is not None: append_config(datadir, extra_conf) # Most callers will just need to add extra args to the standard list below. @@ -89,8 +87,7 @@ class TestNode(): "-debug", "-debugexclude=libevent", "-debugexclude=leveldb", - "-mocktime=" + str(mocktime), - "-uacomment=testnode%d" % i + "-uacomment=testnode%d" % i, ] self.cli = TestNodeCLI(bitcoin_cli, self.datadir) @@ -171,7 +168,7 @@ class TestNode(): assert self.rpc_connected and self.rpc is not None, self._node_msg("Error: no RPC connection") return getattr(self.rpc, name) - def start(self, extra_args=None, *, stdout=None, stderr=None, **kwargs): + def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, **kwargs): """Start the node.""" if extra_args is None: extra_args = self.extra_args @@ -184,6 +181,9 @@ class TestNode(): self.stderr = stderr self.stdout = stdout + if cwd is None: + cwd = self.cwd + # Delete any existing cookie file -- if such a file exists (eg due to # unclean shutdown), it will get overwritten anyway by bitcoind, and # potentially interfere with our attempt to authenticate @@ -192,7 +192,7 @@ class TestNode(): # add environment variable LIBC_FATAL_STDERR_=1 so that libc errors are written to stderr and not the terminal subp_env = dict(os.environ, LIBC_FATAL_STDERR_="1") - self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, **kwargs) + self.process = subprocess.Popen(self.args + extra_args, env=subp_env, stdout=stdout, stderr=stderr, cwd=cwd, **kwargs) self.running = True self.log.debug("bitcoind started, waiting for RPC to come up") @@ -209,12 +209,15 @@ class TestNode(): raise FailedToStartError(self._node_msg( 'bitcoind exited with status {} during initialization'.format(self.process.returncode))) try: - self.rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) - self.rpc.getblockcount() + rpc = get_rpc_proxy(rpc_url(self.datadir, self.index, self.rpchost), self.index, timeout=self.rpc_timeout, coveragedir=self.coverage_dir) + rpc.getblockcount() # If the call to getblockcount() succeeds then the RPC connection is up + self.log.debug("RPC successfully started") + if self.use_cli: + return + self.rpc = rpc self.rpc_connected = True self.url = self.rpc.url - self.log.debug("RPC successfully started") return except IOError as e: if e.errno != errno.ECONNREFUSED: # Port not yet open? @@ -559,5 +562,5 @@ class TestNodeCLI(): raise subprocess.CalledProcessError(returncode, self.binary, output=cli_stderr) try: return json.loads(cli_stdout, parse_float=decimal.Decimal) - except JSONDecodeError: + except json.JSONDecodeError: return cli_stdout.rstrip("\n") diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index d0a78d8dfd..034ed893f4 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helpful routines for regression testing.""" from base64 import b64encode -from binascii import hexlify, unhexlify +from binascii import unhexlify from decimal import Decimal, ROUND_DOWN import hashlib import inspect @@ -182,9 +182,6 @@ def check_json_precision(): def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) -def bytes_to_hex_str(byte_str): - return hexlify(byte_str).decode('ascii') - def hash256(byte_str): sha256 = hashlib.sha256() sha256.update(byte_str) @@ -267,7 +264,7 @@ def get_rpc_proxy(url, node_number, timeout=None, coveragedir=None): return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) def p2p_port(n): - assert(n <= MAX_NODES) + assert n <= MAX_NODES return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def rpc_port(n): @@ -410,12 +407,12 @@ def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True): # Transaction/Block functions ############################# -def find_output(node, txid, amount): +def find_output(node, txid, amount, *, blockhash=None): """ Return index to output of txid with value amount Raises exception if there is none. """ - txdata = node.getrawtransaction(txid, 1) + txdata = node.getrawtransaction(txid, 1, blockhash) for i in range(len(txdata["vout"])): if txdata["vout"][i]["value"] == amount: return i @@ -425,7 +422,7 @@ def gather_inputs(from_node, amount_needed, confirmations_required=1): """ Return a random set of unspent txouts that are enough to pay amount_needed """ - assert(confirmations_required >= 0) + assert confirmations_required >= 0 utxo = from_node.listunspent(confirmations_required) random.shuffle(utxo) inputs = [] @@ -503,7 +500,7 @@ def create_confirmed_utxos(fee, node, count): node.generate(1) utxos = node.listunspent() - assert(len(utxos) >= count) + assert len(utxos) >= count return utxos # Create large OP_RETURN txouts that can be appended to a transaction diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 5c92370b85..a0d9f820e6 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run regression test suite. @@ -7,8 +7,6 @@ This module calls down into individual test cases via subprocess. It will forward all unrecognized arguments onto the individual test scripts. -Functional tests are disabled on Windows by default. Use --force to run them anyway. - For a description of arguments recognized by test scripts, see `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`. @@ -112,14 +110,15 @@ BASE_SCRIPTS = [ 'wallet_txn_clone.py', 'wallet_txn_clone.py --segwit', 'rpc_getchaintips.py', + 'rpc_misc.py', 'interface_rest.py', 'mempool_spend_coinbase.py', 'mempool_reorg.py', 'mempool_persist.py', 'wallet_multiwallet.py', 'wallet_multiwallet.py --usecli', - 'wallet_disableprivatekeys.py', - 'wallet_disableprivatekeys.py --usecli', + 'wallet_createwallet.py', + 'wallet_createwallet.py --usecli', 'interface_http.py', 'interface_rpc.py', 'rpc_psbt.py', @@ -182,6 +181,8 @@ BASE_SCRIPTS = [ 'feature_filelock.py', 'p2p_unrequested_blocks.py', 'feature_includeconf.py', + 'rpc_deriveaddresses.py', + 'rpc_deriveaddresses.py --usecli', 'rpc_scantxoutset.py', 'feature_logging.py', 'p2p_node_network_limited.py', @@ -224,7 +225,6 @@ def main(): parser.add_argument('--ci', action='store_true', help='Run checks and code that are usually only enabled in a continuous integration environment') parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.') parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests') - parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).') parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit') parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') @@ -251,22 +251,12 @@ def main(): # Create base test directory tmpdir = "%s/test_runner_₿_🏃_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) - # If we fixed the command-line and filename encoding issue on Windows, these two lines could be removed - if config["environment"]["EXEEXT"] == ".exe": - tmpdir = "%s/test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S")) - os.makedirs(tmpdir) logging.debug("Temporary test directory at %s" % tmpdir) enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") - if config["environment"]["EXEEXT"] == ".exe" and not args.force: - # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 - # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964 - print("Tests currently disabled on Windows by default. Use --force option to enable") - sys.exit(0) - if not enable_bitcoind: print("No functional tests to run.") print("Rerun ./configure with --with-daemon and then make") diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 0c3c247694..b0b151d2d6 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -26,8 +26,7 @@ from test_framework.util import ( class AbandonConflictTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 - # TODO: remove -txindex. Currently required for getrawtransaction call. - self.extra_args = [["-minrelaytxfee=0.00001", "-txindex"], []] + self.extra_args = [["-minrelaytxfee=0.00001"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index bafa556aad..a44aa5a18e 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet can send and receive using all combinations of address types. @@ -54,6 +54,10 @@ from decimal import Decimal import itertools from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import ( + descsum_create, + descsum_check, +) from test_framework.util import ( assert_equal, assert_greater_than, @@ -98,62 +102,62 @@ class AddressTypeTest(BitcoinTestFramework): def test_address(self, node, address, multisig, typ): """Run sanity checks on an address.""" info = self.nodes[node].getaddressinfo(address) - assert(self.nodes[node].validateaddress(address)['isvalid']) + assert self.nodes[node].validateaddress(address)['isvalid'] assert_equal(info.get('solvable'), True) if not multisig and typ == 'legacy': # P2PKH - assert(not info['isscript']) - assert(not info['iswitness']) - assert('pubkey' in info) + assert not info['isscript'] + assert not info['iswitness'] + assert 'pubkey' in info elif not multisig and typ == 'p2sh-segwit': # P2SH-P2WPKH - assert(info['isscript']) - assert(not info['iswitness']) + assert info['isscript'] + assert not info['iswitness'] assert_equal(info['script'], 'witness_v0_keyhash') - assert('pubkey' in info) + assert 'pubkey' in info elif not multisig and typ == 'bech32': # P2WPKH - assert(not info['isscript']) - assert(info['iswitness']) + assert not info['isscript'] + assert info['iswitness'] assert_equal(info['witness_version'], 0) assert_equal(len(info['witness_program']), 40) - assert('pubkey' in info) + assert 'pubkey' in info elif typ == 'legacy': # P2SH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'multisig') - assert(not info['iswitness']) - assert('pubkeys' in info) + assert not info['iswitness'] + assert 'pubkeys' in info elif typ == 'p2sh-segwit': # P2SH-P2WSH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'witness_v0_scripthash') - assert(not info['iswitness']) - assert(info['embedded']['isscript']) + assert not info['iswitness'] + assert info['embedded']['isscript'] assert_equal(info['embedded']['script'], 'multisig') - assert(info['embedded']['iswitness']) + assert info['embedded']['iswitness'] assert_equal(info['embedded']['witness_version'], 0) assert_equal(len(info['embedded']['witness_program']), 64) - assert('pubkeys' in info['embedded']) + assert 'pubkeys' in info['embedded'] elif typ == 'bech32': # P2WSH-multisig - assert(info['isscript']) + assert info['isscript'] assert_equal(info['script'], 'multisig') - assert(info['iswitness']) + assert info['iswitness'] assert_equal(info['witness_version'], 0) assert_equal(len(info['witness_program']), 64) - assert('pubkeys' in info) + assert 'pubkeys' in info else: # Unknown type - assert(False) + assert False def test_desc(self, node, address, multisig, typ, utxo): """Run sanity checks on a descriptor reported by getaddressinfo.""" info = self.nodes[node].getaddressinfo(address) - assert('desc' in info) + assert 'desc' in info assert_equal(info['desc'], utxo['desc']) - assert(self.nodes[node].validateaddress(address)['isvalid']) + assert self.nodes[node].validateaddress(address)['isvalid'] # Use a ridiculously roundabout way to find the key origin info through # the PSBT logic. However, this does test consistency between the PSBT reported @@ -167,27 +171,34 @@ class AddressTypeTest(BitcoinTestFramework): assert_equal(deriv['path'][0], 'm') key_descs[deriv['pubkey']] = '[' + deriv['master_fingerprint'] + deriv['path'][1:] + ']' + deriv['pubkey'] + # Verify the descriptor checksum against the Python implementation + assert descsum_check(info['desc']) + # Verify that stripping the checksum and recreating it using Python roundtrips + assert info['desc'] == descsum_create(info['desc'][:-9]) + # Verify that stripping the checksum and feeding it to getdescriptorinfo roundtrips + assert info['desc'] == self.nodes[0].getdescriptorinfo(info['desc'][:-9])['descriptor'] + if not multisig and typ == 'legacy': # P2PKH - assert_equal(info['desc'], "pkh(%s)" % key_descs[info['pubkey']]) + assert_equal(info['desc'], descsum_create("pkh(%s)" % key_descs[info['pubkey']])) elif not multisig and typ == 'p2sh-segwit': # P2SH-P2WPKH - assert_equal(info['desc'], "sh(wpkh(%s))" % key_descs[info['pubkey']]) + assert_equal(info['desc'], descsum_create("sh(wpkh(%s))" % key_descs[info['pubkey']])) elif not multisig and typ == 'bech32': # P2WPKH - assert_equal(info['desc'], "wpkh(%s)" % key_descs[info['pubkey']]) + assert_equal(info['desc'], descsum_create("wpkh(%s)" % key_descs[info['pubkey']])) elif typ == 'legacy': # P2SH-multisig - assert_equal(info['desc'], "sh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]])) + assert_equal(info['desc'], descsum_create("sh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]]))) elif typ == 'p2sh-segwit': # P2SH-P2WSH-multisig - assert_equal(info['desc'], "sh(wsh(multi(2,%s,%s)))" % (key_descs[info['embedded']['pubkeys'][0]], key_descs[info['embedded']['pubkeys'][1]])) + assert_equal(info['desc'], descsum_create("sh(wsh(multi(2,%s,%s)))" % (key_descs[info['embedded']['pubkeys'][0]], key_descs[info['embedded']['pubkeys'][1]]))) elif typ == 'bech32': # P2WSH-multisig - assert_equal(info['desc'], "wsh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]])) + assert_equal(info['desc'], descsum_create("wsh(multi(2,%s,%s))" % (key_descs[info['pubkeys'][0]], key_descs[info['pubkeys'][1]]))) else: # Unknown type - assert(False) + assert False def test_change_output_type(self, node_sender, destinations, expected_type): txid = self.nodes[node_sender].sendmany(dummy="", amounts=dict.fromkeys(destinations, 0.001)) diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 673eb718e6..0477164b6b 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -11,7 +11,6 @@ from test_framework.util import ( assert_array_result, assert_equal, assert_fee_amount, - assert_greater_than, assert_raises_rpc_error, connect_nodes_bi, sync_blocks, @@ -24,8 +23,6 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.setup_clean_chain = True - # TODO: remove -txindex. Currently required for getrawtransaction call. - self.extra_args = [[], [], ["-txindex"], []] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -86,13 +83,8 @@ class WalletTest(BitcoinTestFramework): assert_equal(txout['value'], 50) # Send 21 BTC from 0 to 2 using sendtoaddress call. - # Locked memory should increase to sign transactions - self.log.info("test getmemoryinfo") - memory_before = self.nodes[0].getmemoryinfo() self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) mempool_txid = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) - memory_after = self.nodes[0].getmemoryinfo() - assert_greater_than(memory_after['locked']['used'], memory_before['locked']['used']) self.log.info("test gettxout (second part)") # utxo spent in mempool should be visible if you exclude mempool @@ -327,12 +319,38 @@ class WalletTest(BitcoinTestFramework): tx_obj = self.nodes[0].gettransaction(txid) assert_equal(tx_obj['amount'], Decimal('-0.0001')) + # General checks for errors from incorrect inputs # This will raise an exception because the amount type is wrong assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") # This will raise an exception since generate does not accept a string assert_raises_rpc_error(-1, "not an integer", self.nodes[0].generate, "2") + # This will raise an exception for the invalid private key format + assert_raises_rpc_error(-5, "Invalid private key encoding", self.nodes[0].importprivkey, "invalid") + + # This will raise an exception for importing an address with the PS2H flag + temp_address = self.nodes[1].getnewaddress() + assert_raises_rpc_error(-5, "Cannot use the p2sh flag with an address - use a script instead", self.nodes[0].importaddress, temp_address, "label", False, True) + + # This will raise an exception for attempting to dump the private key of an address you do not own + assert_raises_rpc_error(-3, "Address does not refer to a key", self.nodes[0].dumpprivkey, temp_address) + + # This will raise an exception for attempting to get the private key of an invalid Bitcoin address + assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].dumpprivkey, "invalid") + + # This will raise an exception for attempting to set a label for an invalid Bitcoin address + assert_raises_rpc_error(-5, "Invalid Bitcoin address", self.nodes[0].setlabel, "invalid address", "label") + + # This will raise an exception for importing an invalid address + assert_raises_rpc_error(-5, "Invalid Bitcoin address or script", self.nodes[0].importaddress, "invalid") + + # This will raise an exception for attempting to import a pubkey that isn't in hex + assert_raises_rpc_error(-5, "Pubkey must be a hex string", self.nodes[0].importpubkey, "not hex") + + # This will raise an exception for importing an invalid pubkey + assert_raises_rpc_error(-5, "Pubkey is not a valid public key", self.nodes[0].importpubkey, "5361746f736869204e616b616d6f746f") + # Import address and private key to check correct behavior of spendable unspents # 1. Send some coins to generate new UTXO address_to_import = self.nodes[2].getnewaddress() diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 7d3d9b61e2..fc752e5ac0 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the bumpfee RPC. @@ -19,7 +19,7 @@ import io from test_framework.blocktools import add_witness_commitment, create_block, create_coinbase, send_to_witness from test_framework.messages import BIP125_SEQUENCE_NUMBER, CTransaction from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, bytes_to_hex_str, connect_nodes_bi, hex_str_to_bytes, sync_mempools +from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, connect_nodes_bi, hex_str_to_bytes, sync_mempools WALLET_PASSPHRASE = "test" WALLET_PASSPHRASE_TIMEOUT = 3600 @@ -298,7 +298,7 @@ def submit_block_with_tx(node, tx): block.hashMerkleRoot = block.calc_merkle_root() add_witness_commitment(block) block.solve() - node.submitblock(bytes_to_hex_str(block.serialize(True))) + node.submitblock(block.serialize(True).hex()) return block diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index 27dc0fb279..0b584a0bb2 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,17 +7,25 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, ) +from test_framework.blocktools import ( + TIME_GENESIS_BLOCK, +) class CreateTxWalletTest(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = False + self.setup_clean_chain = True self.num_nodes = 1 def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): + self.log.info('Create some old blocks') + self.nodes[0].setmocktime(TIME_GENESIS_BLOCK) + self.nodes[0].generate(200) + self.nodes[0].setmocktime(0) + self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled') assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py new file mode 100755 index 0000000000..8ec4b98b6e --- /dev/null +++ b/test/functional/wallet_createwallet.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test createwallet arguments. +""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) + +class CreateWalletTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + self.supports_cli = True + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + node = self.nodes[0] + node.generate(1) # Leave IBD for sethdseed + + self.nodes[0].createwallet(wallet_name='w0') + w0 = node.get_wallet_rpc('w0') + address1 = w0.getnewaddress() + + self.log.info("Test disableprivatekeys creation.") + self.nodes[0].createwallet(wallet_name='w1', disable_private_keys=True) + w1 = node.get_wallet_rpc('w1') + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w1.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w1.getrawchangeaddress) + w1.importpubkey(w0.getaddressinfo(address1)['pubkey']) + + self.log.info('Test that private keys cannot be imported') + addr = w0.getnewaddress('', 'legacy') + privkey = w0.dumpprivkey(addr) + assert_raises_rpc_error(-4, 'Cannot import private keys to a wallet with private keys disabled', w1.importprivkey, privkey) + result = w1.importmulti([{'scriptPubKey': {'address': addr}, 'timestamp': 'now', 'keys': [privkey]}]) + assert not result[0]['success'] + assert 'warning' not in result[0] + assert_equal(result[0]['error']['code'], -4) + assert_equal(result[0]['error']['message'], 'Cannot import private keys to a wallet with private keys disabled') + + self.log.info("Test blank creation with private keys disabled.") + self.nodes[0].createwallet(wallet_name='w2', disable_private_keys=True, blank=True) + w2 = node.get_wallet_rpc('w2') + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w2.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w2.getrawchangeaddress) + w2.importpubkey(w0.getaddressinfo(address1)['pubkey']) + + self.log.info("Test blank creation with private keys enabled.") + self.nodes[0].createwallet(wallet_name='w3', disable_private_keys=False, blank=True) + w3 = node.get_wallet_rpc('w3') + assert_equal(w3.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getrawchangeaddress) + # Import private key + w3.importprivkey(w0.dumpprivkey(address1)) + # Imported private keys are currently ignored by the keypool + assert_equal(w3.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w3.getnewaddress) + # Set the seed + w3.sethdseed() + assert_equal(w3.getwalletinfo()['keypoolsize'], 1) + w3.getnewaddress() + w3.getrawchangeaddress() + + self.log.info("Test blank creation with privkeys enabled and then encryption") + self.nodes[0].createwallet(wallet_name='w4', disable_private_keys=False, blank=True) + w4 = node.get_wallet_rpc('w4') + assert_equal(w4.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress) + # Encrypt the wallet. Nothing should change about the keypool + w4.encryptwallet('pass') + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w4.getrawchangeaddress) + # Now set a seed and it should work. Wallet should also be encrypted + w4.walletpassphrase('pass', 2) + w4.sethdseed() + w4.getnewaddress() + w4.getrawchangeaddress() + + self.log.info("Test blank creation with privkeys disabled and then encryption") + self.nodes[0].createwallet(wallet_name='w5', disable_private_keys=True, blank=True) + w5 = node.get_wallet_rpc('w5') + assert_equal(w5.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getrawchangeaddress) + # Encrypt the wallet + w5.encryptwallet('pass') + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getnewaddress) + assert_raises_rpc_error(-4, "Error: This wallet has no available keys", w5.getrawchangeaddress) + +if __name__ == '__main__': + CreateWalletTest().main() diff --git a/test/functional/wallet_disable.py b/test/functional/wallet_disable.py index 6530c58c78..7c2ec56b5a 100755 --- a/test/functional/wallet_disable.py +++ b/test/functional/wallet_disable.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test a node with the -disablewallet option. @@ -21,9 +21,9 @@ class DisableWalletTest (BitcoinTestFramework): # Make sure wallet is really disabled assert_raises_rpc_error(-32601, 'Method not found', self.nodes[0].getwalletinfo) x = self.nodes[0].validateaddress('3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') - assert(x['isvalid'] == False) + assert x['isvalid'] == False x = self.nodes[0].validateaddress('mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') - assert(x['isvalid'] == True) + assert x['isvalid'] == True # Checking mining to an address without a wallet. Generating to a valid address should succeed # but generating to an invalid address will fail. diff --git a/test/functional/wallet_disableprivatekeys.py b/test/functional/wallet_disableprivatekeys.py deleted file mode 100755 index e55bb82e76..0000000000 --- a/test/functional/wallet_disableprivatekeys.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test disable-privatekeys mode. -""" - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - assert_raises_rpc_error, -) - - -class DisablePrivateKeysTest(BitcoinTestFramework): - def set_test_params(self): - self.setup_clean_chain = False - self.num_nodes = 1 - self.supports_cli = True - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - - def run_test(self): - node = self.nodes[0] - self.log.info("Test disableprivatekeys creation.") - self.nodes[0].createwallet('w1', True) - self.nodes[0].createwallet('w2') - w1 = node.get_wallet_rpc('w1') - w2 = node.get_wallet_rpc('w2') - assert_raises_rpc_error(-4,"Error: Private keys are disabled for this wallet", w1.getnewaddress) - assert_raises_rpc_error(-4,"Error: Private keys are disabled for this wallet", w1.getrawchangeaddress) - w1.importpubkey(w2.getaddressinfo(w2.getnewaddress())['pubkey']) - - self.log.info('Test that private keys cannot be imported') - addr = w2.getnewaddress('', 'legacy') - privkey = w2.dumpprivkey(addr) - assert_raises_rpc_error(-4, 'Cannot import private keys to a wallet with private keys disabled', w1.importprivkey, privkey) - result = w1.importmulti([{'scriptPubKey': {'address': addr}, 'timestamp': 'now', 'keys': [privkey]}]) - assert(not result[0]['success']) - assert('warning' not in result[0]) - assert_equal(result[0]['error']['code'], -4) - assert_equal(result[0]['error']['message'], 'Cannot import private keys to a wallet with private keys disabled') - -if __name__ == '__main__': - DisablePrivateKeysTest().main() diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 3f39654bb8..53edf710b9 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" @@ -46,10 +46,10 @@ def read_dump(file_name, addrs, script_addrs, hd_master_addr_old): keypath = None if keytype == "inactivehdseed=1": # ensure the old master is still available - assert (hd_master_addr_old == addr) + assert hd_master_addr_old == addr elif keytype == "hdseed=1": # ensure we have generated a new hd master key - assert (hd_master_addr_old != addr) + assert hd_master_addr_old != addr hd_master_addr_ret = addr elif keytype == "script=1": # scripts don't have keypaths diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index eb42531693..97172d8b82 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2018 The Bitcoin Core developers +# Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Hierarchical Deterministic wallet function.""" @@ -27,7 +27,6 @@ class WalletHDTest(BitcoinTestFramework): def run_test(self): # Make sure we use hd, keep masterkeyid masterkeyid = self.nodes[1].getwalletinfo()['hdseedid'] - assert_equal(masterkeyid, self.nodes[1].getwalletinfo()['hdmasterkeyid']) assert_equal(len(masterkeyid), 40) # create an internal key @@ -53,7 +52,6 @@ class WalletHDTest(BitcoinTestFramework): hd_info = self.nodes[1].getaddressinfo(hd_add) assert_equal(hd_info["hdkeypath"], "m/0'/0'/"+str(i)+"'") assert_equal(hd_info["hdseedid"], masterkeyid) - assert_equal(hd_info["hdmasterkeyid"], masterkeyid) self.nodes[0].sendtoaddress(hd_add, 1) self.nodes[0].generate(1) self.nodes[0].sendtoaddress(non_hd_add, 1) @@ -83,7 +81,6 @@ class WalletHDTest(BitcoinTestFramework): hd_info_2 = self.nodes[1].getaddressinfo(hd_add_2) assert_equal(hd_info_2["hdkeypath"], "m/0'/0'/"+str(i)+"'") assert_equal(hd_info_2["hdseedid"], masterkeyid) - assert_equal(hd_info_2["hdmasterkeyid"], masterkeyid) assert_equal(hd_add, hd_add_2) connect_nodes_bi(self.nodes, 0, 1) self.sync_all() diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index f122f19e3a..5bfbaa2f0b 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importmulti RPC. @@ -20,11 +20,11 @@ from test_framework.script import ( OP_NOP, ) from test_framework.test_framework import BitcoinTestFramework +from test_framework.descriptors import descsum_create from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, - bytes_to_hex_str, ) from test_framework.wallet_util import ( get_key, @@ -126,7 +126,7 @@ class ImportMultiTest(BitcoinTestFramework): # Nonstandard scriptPubKey + !internal self.log.info("Should not import a nonstandard scriptPubKey without internal flag") - nonstandardScriptPubKey = key.p2pkh_script + bytes_to_hex_str(CScript([OP_NOP])) + nonstandardScriptPubKey = key.p2pkh_script + CScript([OP_NOP]).hex() key = get_key(self.nodes[0]) self.test_importmulti({"scriptPubKey": nonstandardScriptPubKey, "timestamp": "now"}, @@ -203,7 +203,7 @@ class ImportMultiTest(BitcoinTestFramework): "keys": [key.privkey]}, success=False, error_code=-4, - error_message='The wallet already contains the private key for this address or script') + error_message='The wallet already contains the private key for this address or script ("' + key.p2pkh_script + '")') # Address + Private key + watchonly self.log.info("Should import an address with private key and with watchonly") @@ -543,5 +543,275 @@ class ImportMultiTest(BitcoinTestFramework): solvable=True, ismine=False) + # Test importing of a P2SH-P2WPKH address via descriptor + private key + key = get_key(self.nodes[0]) + self.log.info("Should not import a p2sh-p2wpkh address from descriptor without checksum and private key") + self.test_importmulti({"desc": "sh(wpkh(" + key.pubkey + "))", + "timestamp": "now", + "label": "Descriptor import test", + "keys": [key.privkey]}, + success=False, + error_code=-5, + error_message="Descriptor is invalid") + + # Test importing of a P2SH-P2WPKH address via descriptor + private key + key = get_key(self.nodes[0]) + self.log.info("Should import a p2sh-p2wpkh address from descriptor and private key") + self.test_importmulti({"desc": descsum_create("sh(wpkh(" + key.pubkey + "))"), + "timestamp": "now", + "label": "Descriptor import test", + "keys": [key.privkey]}, + success=True) + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True, + ismine=True, + label="Descriptor import test") + + # Test ranged descriptor fails if range is not specified + xpriv = "tprv8ZgxMBicQKsPeuVhWwi6wuMQGfPKi9Li5GtX35jVNknACgqe3CY4g5xgkfDDJcmtF7o1QnxWDRYw4H5P26PXq7sbcUkEqeR4fg3Kxp2tigg" + addresses = ["2N7yv4p8G8yEaPddJxY41kPihnWvs39qCMf", "2MsHxyb2JS3pAySeNUsJ7mNnurtpeenDzLA"] # hdkeypath=m/0'/0'/0' and 1' + desc = "sh(wpkh(" + xpriv + "/0'/0'/*'" + "))" + self.log.info("Ranged descriptor import should fail without a specified range") + self.test_importmulti({"desc": descsum_create(desc), + "timestamp": "now"}, + success=False, + error_code=-8, + error_message='Descriptor is ranged, please specify the range') + + # Test importing of a ranged descriptor without keys + self.log.info("Should import the ranged descriptor with specified range as solvable") + self.test_importmulti({"desc": descsum_create(desc), + "timestamp": "now", + "range": 1}, + success=True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + for address in addresses: + test_address(self.nodes[1], + key.p2sh_p2wpkh_addr, + solvable=True) + + # Test importing of a P2PKH address via descriptor + key = get_key(self.nodes[0]) + self.log.info("Should import a p2pkh address from descriptor") + self.test_importmulti({"desc": descsum_create("pkh(" + key.pubkey + ")"), + "timestamp": "now", + "label": "Descriptor import test"}, + True, + warnings=["Some private keys are missing, outputs will be considered watchonly. If this is intentional, specify the watchonly flag."]) + test_address(self.nodes[1], + key.p2pkh_addr, + solvable=True, + ismine=False, + label="Descriptor import test") + + # Test import fails if both desc and scriptPubKey are provided + key = get_key(self.nodes[0]) + self.log.info("Import should fail if both scriptPubKey and desc are provided") + self.test_importmulti({"desc": descsum_create("pkh(" + key.pubkey + ")"), + "scriptPubKey": {"address": key.p2pkh_addr}, + "timestamp": "now"}, + success=False, + error_code=-8, + error_message='Both a descriptor and a scriptPubKey should not be provided.') + + # Test import fails if neither desc nor scriptPubKey are present + key = get_key(self.nodes[0]) + self.log.info("Import should fail if neither a descriptor nor a scriptPubKey are provided") + self.test_importmulti({"timestamp": "now"}, + success=False, + error_code=-8, + error_message='Either a descriptor or scriptPubKey must be provided.') + + # Test importing of a multisig via descriptor + key1 = get_key(self.nodes[0]) + key2 = get_key(self.nodes[0]) + self.log.info("Should import a 1-of-2 bare multisig from descriptor") + self.test_importmulti({"desc": descsum_create("multi(1," + key1.pubkey + "," + key2.pubkey + ")"), + "timestamp": "now"}, + success=True) + self.log.info("Should not treat individual keys from the imported bare multisig as watchonly") + test_address(self.nodes[1], + key1.p2pkh_addr, + ismine=False, + iswatchonly=False) + + # Import pubkeys with key origin info + self.log.info("Addresses should have hd keypath and master key id after import with key origin") + pub_addr = self.nodes[1].getnewaddress() + pub_addr = self.nodes[1].getnewaddress() + info = self.nodes[1].getaddressinfo(pub_addr) + pub = info['pubkey'] + pub_keypath = info['hdkeypath'] + pub_fpr = info['hdmasterfingerprint'] + result = self.nodes[0].importmulti( + [{ + 'desc' : descsum_create("wpkh([" + pub_fpr + pub_keypath[1:] +"]" + pub + ")"), + "timestamp": "now", + }] + ) + assert result[0]['success'] + pub_import_info = self.nodes[0].getaddressinfo(pub_addr) + assert_equal(pub_import_info['hdmasterfingerprint'], pub_fpr) + assert_equal(pub_import_info['pubkey'], pub) + assert_equal(pub_import_info['hdkeypath'], pub_keypath) + + # Import privkeys with key origin info + priv_addr = self.nodes[1].getnewaddress() + info = self.nodes[1].getaddressinfo(priv_addr) + priv = self.nodes[1].dumpprivkey(priv_addr) + priv_keypath = info['hdkeypath'] + priv_fpr = info['hdmasterfingerprint'] + result = self.nodes[0].importmulti( + [{ + 'desc' : descsum_create("wpkh([" + priv_fpr + priv_keypath[1:] + "]" + priv + ")"), + "timestamp": "now", + }] + ) + assert result[0]['success'] + priv_import_info = self.nodes[0].getaddressinfo(priv_addr) + assert_equal(priv_import_info['hdmasterfingerprint'], priv_fpr) + assert_equal(priv_import_info['hdkeypath'], priv_keypath) + + # Make sure the key origin info are still there after a restart + self.stop_nodes() + self.start_nodes() + import_info = self.nodes[0].getaddressinfo(pub_addr) + assert_equal(import_info['hdmasterfingerprint'], pub_fpr) + assert_equal(import_info['hdkeypath'], pub_keypath) + import_info = self.nodes[0].getaddressinfo(priv_addr) + assert_equal(import_info['hdmasterfingerprint'], priv_fpr) + assert_equal(import_info['hdkeypath'], priv_keypath) + + # Check legacy import does not import key origin info + self.log.info("Legacy imports don't have key origin info") + pub_addr = self.nodes[1].getnewaddress() + info = self.nodes[1].getaddressinfo(pub_addr) + pub = info['pubkey'] + result = self.nodes[0].importmulti( + [{ + 'scriptPubKey': {'address': pub_addr}, + 'pubkeys': [pub], + "timestamp": "now", + }] + ) + assert result[0]['success'] + pub_import_info = self.nodes[0].getaddressinfo(pub_addr) + assert_equal(pub_import_info['pubkey'], pub) + assert 'hdmasterfingerprint' not in pub_import_info + assert 'hdkeypath' not in pub_import_info + + # Import some public keys to the keypool of a no privkey wallet + self.log.info("Adding pubkey to keypool of disableprivkey wallet") + self.nodes[1].createwallet(wallet_name="noprivkeys", disable_private_keys=True) + wrpc = self.nodes[1].get_wallet_rpc("noprivkeys") + + addr1 = self.nodes[0].getnewaddress() + addr2 = self.nodes[0].getnewaddress() + pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] + pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] + result = wrpc.importmulti( + [{ + 'desc': descsum_create('wpkh(' + pub1 + ')'), + 'keypool': True, + "timestamp": "now", + }, + { + 'desc': descsum_create('wpkh(' + pub2 + ')'), + 'keypool': True, + "timestamp": "now", + }] + ) + assert result[0]['success'] + assert result[1]['success'] + assert_equal(wrpc.getwalletinfo()["keypoolsize"], 2) + newaddr1 = wrpc.getnewaddress() + assert_equal(addr1, newaddr1) + newaddr2 = wrpc.getnewaddress() + assert_equal(addr2, newaddr2) + + # Import some public keys to the internal keypool of a no privkey wallet + self.log.info("Adding pubkey to internal keypool of disableprivkey wallet") + addr1 = self.nodes[0].getnewaddress() + addr2 = self.nodes[0].getnewaddress() + pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] + pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] + result = wrpc.importmulti( + [{ + 'desc': descsum_create('wpkh(' + pub1 + ')'), + 'keypool': True, + 'internal': True, + "timestamp": "now", + }, + { + 'desc': descsum_create('wpkh(' + pub2 + ')'), + 'keypool': True, + 'internal': True, + "timestamp": "now", + }] + ) + assert result[0]['success'] + assert result[1]['success'] + assert_equal(wrpc.getwalletinfo()["keypoolsize_hd_internal"], 2) + newaddr1 = wrpc.getrawchangeaddress() + assert_equal(addr1, newaddr1) + newaddr2 = wrpc.getrawchangeaddress() + assert_equal(addr2, newaddr2) + + # Import a multisig and make sure the keys don't go into the keypool + self.log.info('Imported scripts with pubkeys shoud not have their pubkeys go into the keypool') + addr1 = self.nodes[0].getnewaddress() + addr2 = self.nodes[0].getnewaddress() + pub1 = self.nodes[0].getaddressinfo(addr1)['pubkey'] + pub2 = self.nodes[0].getaddressinfo(addr2)['pubkey'] + result = wrpc.importmulti( + [{ + 'desc': descsum_create('wsh(multi(2,' + pub1 + ',' + pub2 + '))'), + 'keypool': True, + "timestamp": "now", + }] + ) + assert result[0]['success'] + assert_equal(wrpc.getwalletinfo()["keypoolsize"], 0) + + # Cannot import those pubkeys to keypool of wallet with privkeys + self.log.info("Pubkeys cannot be added to the keypool of a wallet with private keys") + wrpc = self.nodes[1].get_wallet_rpc("") + assert wrpc.getwalletinfo()['private_keys_enabled'] + result = wrpc.importmulti( + [{ + 'desc': descsum_create('wpkh(' + pub1 + ')'), + 'keypool': True, + "timestamp": "now", + }] + ) + assert_equal(result[0]['error']['code'], -8) + assert_equal(result[0]['error']['message'], "Keys can only be imported to the keypool when private keys are disabled") + + # Make sure ranged imports import keys in order + self.log.info('Key ranges should be imported in order') + wrpc = self.nodes[1].get_wallet_rpc("noprivkeys") + assert_equal(wrpc.getwalletinfo()["keypoolsize"], 0) + assert_equal(wrpc.getwalletinfo()["private_keys_enabled"], False) + xpub = "tpubDAXcJ7s7ZwicqjprRaEWdPoHKrCS215qxGYxpusRLLmJuT69ZSicuGdSfyvyKpvUNYBW1s2U3NSrT6vrCYB9e6nZUEvrqnwXPF8ArTCRXMY" + addresses = [ + 'bcrt1qtmp74ayg7p24uslctssvjm06q5phz4yrxucgnv', # m/0'/0'/0 + 'bcrt1q8vprchan07gzagd5e6v9wd7azyucksq2xc76k8', # m/0'/0'/1 + 'bcrt1qtuqdtha7zmqgcrr26n2rqxztv5y8rafjp9lulu', # m/0'/0'/2 + 'bcrt1qau64272ymawq26t90md6an0ps99qkrse58m640', # m/0'/0'/3 + 'bcrt1qsg97266hrh6cpmutqen8s4s962aryy77jp0fg0', # m/0'/0'/4 + ] + result = wrpc.importmulti( + [{ + 'desc': descsum_create('wpkh([80002067/0h/0h]' + xpub + '/*)'), + 'keypool': True, + 'timestamp': 'now', + 'range' : [0, 4], + }] + ) + for i in range(0, 5): + addr = wrpc.getnewaddress('', 'bech32') + assert_equal(addr, addresses[i]) + if __name__ == '__main__': ImportMultiTest().main() diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index ceb9709712..e3aeb61197 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet keypool and interaction with wallet encryption/locking.""" @@ -21,8 +21,7 @@ class KeyPoolTest(BitcoinTestFramework): addr_before_encrypting = nodes[0].getnewaddress() addr_before_encrypting_data = nodes[0].getaddressinfo(addr_before_encrypting) wallet_info_old = nodes[0].getwalletinfo() - assert_equal(wallet_info_old['hdseedid'], wallet_info_old['hdmasterkeyid']) - assert(addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid']) + assert addr_before_encrypting_data['hdseedid'] == wallet_info_old['hdseedid'] # Encrypt wallet and wait to terminate nodes[0].encryptwallet('test') @@ -30,9 +29,8 @@ class KeyPoolTest(BitcoinTestFramework): addr = nodes[0].getnewaddress() addr_data = nodes[0].getaddressinfo(addr) wallet_info = nodes[0].getwalletinfo() - assert_equal(wallet_info['hdseedid'], wallet_info['hdmasterkeyid']) - assert(addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid']) - assert(addr_data['hdseedid'] == wallet_info['hdseedid']) + assert addr_before_encrypting_data['hdseedid'] != wallet_info['hdseedid'] + assert addr_data['hdseedid'] == wallet_info['hdseedid'] assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) # put six (plus 2) new keys in the keypool (100% external-, +100% internal-keys, 1 in min) @@ -61,7 +59,7 @@ class KeyPoolTest(BitcoinTestFramework): addr.add(nodes[0].getnewaddress()) addr.add(nodes[0].getnewaddress()) addr.add(nodes[0].getnewaddress()) - assert(len(addr) == 6) + assert len(addr) == 6 # the next one should fail assert_raises_rpc_error(-12, "Error: Keypool ran out, please call keypoolrefill first", nodes[0].getnewaddress) diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index b7c8d3098d..e099bab6a3 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test HD Wallet keypool restore function. @@ -55,11 +55,11 @@ class KeypoolRestoreTest(BitcoinTestFramework): # Make sure we're creating the outputs we expect address_details = self.nodes[idx].validateaddress(addr_extpool) if i == 0: - assert(not address_details["isscript"] and not address_details["iswitness"]) + assert not address_details["isscript"] and not address_details["iswitness"] elif i == 1: - assert(address_details["isscript"] and not address_details["iswitness"]) + assert address_details["isscript"] and not address_details["iswitness"] else: - assert(not address_details["isscript"] and address_details["iswitness"]) + assert not address_details["isscript"] and address_details["iswitness"] self.log.info("Send funds to wallet") diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 17f044bf65..f8268fab35 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listtransactions API.""" @@ -11,7 +11,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_array_result, assert_equal, - bytes_to_hex_str, hex_str_to_bytes, sync_mempools, ) @@ -126,7 +125,7 @@ class ListTransactionsTest(BitcoinTestFramework): # 1. Chain a few transactions that don't opt-in. txid_1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) - assert(not is_opt_in(self.nodes[0], txid_1)) + assert not is_opt_in(self.nodes[0], txid_1) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_1}, {"bip125-replaceable": "no"}) sync_mempools(self.nodes) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_1}, {"bip125-replaceable": "no"}) @@ -146,7 +145,7 @@ class ListTransactionsTest(BitcoinTestFramework): txid_2 = self.nodes[1].sendrawtransaction(tx2_signed) # ...and check the result - assert(not is_opt_in(self.nodes[1], txid_2)) + assert not is_opt_in(self.nodes[1], txid_2) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_2}, {"bip125-replaceable": "no"}) sync_mempools(self.nodes) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_2}, {"bip125-replaceable": "no"}) @@ -158,11 +157,11 @@ class ListTransactionsTest(BitcoinTestFramework): tx3 = self.nodes[0].createrawtransaction(inputs, outputs) tx3_modified = tx_from_hex(tx3) tx3_modified.vin[0].nSequence = 0 - tx3 = bytes_to_hex_str(tx3_modified.serialize()) + tx3 = tx3_modified.serialize().hex() tx3_signed = self.nodes[0].signrawtransactionwithwallet(tx3)['hex'] txid_3 = self.nodes[0].sendrawtransaction(tx3_signed) - assert(is_opt_in(self.nodes[0], txid_3)) + assert is_opt_in(self.nodes[0], txid_3) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_3}, {"bip125-replaceable": "yes"}) sync_mempools(self.nodes) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_3}, {"bip125-replaceable": "yes"}) @@ -176,7 +175,7 @@ class ListTransactionsTest(BitcoinTestFramework): tx4_signed = self.nodes[1].signrawtransactionwithwallet(tx4)["hex"] txid_4 = self.nodes[1].sendrawtransaction(tx4_signed) - assert(not is_opt_in(self.nodes[1], txid_4)) + assert not is_opt_in(self.nodes[1], txid_4) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "yes"}) sync_mempools(self.nodes) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "yes"}) @@ -184,10 +183,10 @@ class ListTransactionsTest(BitcoinTestFramework): # Replace tx3, and check that tx4 becomes unknown tx3_b = tx3_modified tx3_b.vout[0].nValue -= int(Decimal("0.004") * COIN) # bump the fee - tx3_b = bytes_to_hex_str(tx3_b.serialize()) + tx3_b = tx3_b.serialize().hex() tx3_b_signed = self.nodes[0].signrawtransactionwithwallet(tx3_b)['hex'] txid_3b = self.nodes[0].sendrawtransaction(tx3_b_signed, True) - assert(is_opt_in(self.nodes[0], txid_3b)) + assert is_opt_in(self.nodes[0], txid_3b) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "unknown"}) sync_mempools(self.nodes) @@ -203,7 +202,7 @@ class ListTransactionsTest(BitcoinTestFramework): # After mining a transaction, it's no longer BIP125-replaceable self.nodes[0].generate(1) - assert(txid_3b not in self.nodes[0].getrawmempool()) + assert txid_3b not in self.nodes[0].getrawmempool() assert_equal(self.nodes[0].gettransaction(txid_3b)["bip125-replaceable"], "no") assert_equal(self.nodes[0].gettransaction(txid_4)["bip125-replaceable"], "unknown") diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index df778f57df..984ffab5a4 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiwallet. diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index 1c2e0a9cb7..af1b321a07 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there are cloned transactions with malleated scriptsigs.""" +import io from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -11,6 +12,7 @@ from test_framework.util import ( disconnect_nodes, sync_blocks, ) +from test_framework.messages import CTransaction, COIN class TxnMallTest(BitcoinTestFramework): def set_test_params(self): @@ -72,19 +74,14 @@ class TxnMallTest(BitcoinTestFramework): clone_raw = self.nodes[0].createrawtransaction(clone_inputs, clone_outputs, clone_locktime) # createrawtransaction randomizes the order of its outputs, so swap them if necessary. - # output 0 is at version+#inputs+input+sigstub+sequence+#outputs - # 40 BTC serialized is 00286bee00000000 - pos0 = 2 * (4 + 1 + 36 + 1 + 4 + 1) - hex40 = "00286bee00000000" - output_len = 16 + 2 + 2 * int("0x" + clone_raw[pos0 + 16:pos0 + 16 + 2], 0) - if (rawtx1["vout"][0]["value"] == 40 and clone_raw[pos0:pos0 + 16] != hex40 or rawtx1["vout"][0]["value"] != 40 and clone_raw[pos0:pos0 + 16] == hex40): - output0 = clone_raw[pos0:pos0 + output_len] - output1 = clone_raw[pos0 + output_len:pos0 + 2 * output_len] - clone_raw = clone_raw[:pos0] + output1 + output0 + clone_raw[pos0 + 2 * output_len:] + clone_tx = CTransaction() + clone_tx.deserialize(io.BytesIO(bytes.fromhex(clone_raw))) + if (rawtx1["vout"][0]["value"] == 40 and clone_tx.vout[0].nValue != 40*COIN or rawtx1["vout"][0]["value"] != 40 and clone_tx.vout[0].nValue == 40*COIN): + (clone_tx.vout[0], clone_tx.vout[1]) = (clone_tx.vout[1], clone_tx.vout[0]) # Use a different signature hash type to sign. This creates an equivalent but malleated clone. # Don't send the clone anywhere yet - tx1_clone = self.nodes[0].signrawtransactionwithwallet(clone_raw, None, "ALL|ANYONECANPAY") + tx1_clone = self.nodes[0].signrawtransactionwithwallet(clone_tx.serialize().hex(), None, "ALL|ANYONECANPAY") assert_equal(tx1_clone["complete"], True) # Have node0 mine a block, if requested: diff --git a/test/functional/wallet_txn_doublespend.py b/test/functional/wallet_txn_doublespend.py index f114d5ab68..8dcfd74f56 100755 --- a/test/functional/wallet_txn_doublespend.py +++ b/test/functional/wallet_txn_doublespend.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there is a double-spend conflict.""" @@ -34,6 +34,14 @@ class TxnMallTest(BitcoinTestFramework): def run_test(self): # All nodes should start with 1,250 BTC: starting_balance = 1250 + + # All nodes should be out of IBD. + # If the nodes are not all out of IBD, that can interfere with + # blockchain sync later in the test when nodes are connected, due to + # timing issues. + for n in self.nodes: + assert n.getblockchaininfo()["initialblockdownload"] == False + for i in range(4): assert_equal(self.nodes[i].getbalance(), starting_balance) self.nodes[i].getnewaddress("") # bug workaround, coins generated assigned to first getnewaddress! diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py new file mode 100755 index 0000000000..1869f71753 --- /dev/null +++ b/test/fuzz/test_runner.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Run fuzz test targets. +""" + +import argparse +import configparser +import os +import sys +import subprocess +import logging + + +def main(): + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + "-l", + "--loglevel", + dest="loglevel", + default="INFO", + help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.", + ) + parser.add_argument( + '--export_coverage', + action='store_true', + help='If true, export coverage information to files in the seed corpus', + ) + parser.add_argument( + 'seed_dir', + help='The seed corpus to run on (must contain subfolders for each fuzz target).', + ) + parser.add_argument( + 'target', + nargs='*', + help='The target(s) to run. Default is to run all targets.', + ) + + args = parser.parse_args() + + # Set up logging + logging.basicConfig( + format='%(message)s', + level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(), + ) + + # Read config generated by configure. + config = configparser.ConfigParser() + configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" + config.read_file(open(configfile, encoding="utf8")) + + if not config["components"].getboolean("ENABLE_FUZZ"): + logging.error("Must have fuzz targets built") + sys.exit(1) + + # Build list of tests + test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include')) + + if not test_list_all: + logging.error("No fuzz targets found") + sys.exit(1) + + logging.info("Fuzz targets found: {}".format(test_list_all)) + + args.target = args.target or test_list_all # By default run all + test_list_error = list(set(args.target).difference(set(test_list_all))) + if test_list_error: + logging.error("Unknown fuzz targets selected: {}".format(test_list_error)) + test_list_selection = list(set(test_list_all).intersection(set(args.target))) + if not test_list_selection: + logging.error("No fuzz targets selected") + logging.info("Fuzz targets selected: {}".format(test_list_selection)) + + try: + help_output = subprocess.run( + args=[ + os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]), + '-help=1', + ], + timeout=1, + check=True, + stderr=subprocess.PIPE, + universal_newlines=True, + ).stderr + if "libFuzzer" not in help_output: + logging.error("Must be built with libFuzzer") + sys.exit(1) + except subprocess.TimeoutExpired: + logging.error("subprocess timed out: Currently only libFuzzer is supported") + sys.exit(1) + + run_once( + corpus=args.seed_dir, + test_list=test_list_selection, + build_dir=config["environment"]["BUILDDIR"], + export_coverage=args.export_coverage, + ) + + +def run_once(*, corpus, test_list, build_dir, export_coverage): + for t in test_list: + args = [ + os.path.join(build_dir, 'src', 'test', 'fuzz', t), + '-runs=1', + os.path.join(corpus, t), + ] + logging.debug('Run {} with args {}'.format(t, args)) + output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr + logging.debug('Output: {}'.format(output)) + if not export_coverage: + continue + for l in output.splitlines(): + if 'INITED' in l: + with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file: + cov_file.write(l) + break + + +def parse_test_list(makefile): + with open(makefile, encoding='utf-8') as makefile_test: + test_list_all = [] + read_targets = False + for line in makefile_test.readlines(): + line = line.strip().replace('test/fuzz/', '').replace(' \\', '') + if read_targets: + if not line: + break + test_list_all.append(line) + continue + + if line == 'FUZZ_TARGETS =': + read_targets = True + return test_list_all + + +if __name__ == '__main__': + main() diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index c370ce0c04..3b05d5055c 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index f5d4780b6d..224e62f04a 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -39,7 +39,7 @@ def parse_function_calls(function_name, source_code): >>> len(parse_function_calls("foo", "#define FOO foo();")) 0 """ - assert(type(function_name) is str and type(source_code) is str and function_name) + assert type(function_name) is str and type(source_code) is str and function_name lines = [re.sub("// .*", " ", line).strip() for line in source_code.split("\n") if not line.strip().startswith("#")] @@ -53,7 +53,7 @@ def normalize(s): >>> normalize(" /* nothing */ foo\tfoo /* bar */ foo ") 'foo foo foo' """ - assert(type(s) is str) + assert type(s) is str s = s.replace("\n", " ") s = s.replace("\t", " ") s = re.sub("/\*.*?\*/", " ", s) @@ -77,7 +77,7 @@ def escape(s): >>> escape(r'foo \\t foo \\n foo \\\\ foo \\ foo \\"bar\\"') 'foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]' """ - assert(type(s) is str) + assert type(s) is str for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(raw_value, escaped_value) return s @@ -92,7 +92,7 @@ def unescape(s): >>> unescape("foo [escaped-tab] foo [escaped-newline] foo \\\\\\\\ foo \\\\ foo [escaped-quote]bar[escaped-quote]") 'foo \\\\t foo \\\\n foo \\\\\\\\ foo \\\\ foo \\\\"bar\\\\"' """ - assert(type(s) is str) + assert type(s) is str for raw_value, escaped_value in ESCAPE_MAP.items(): s = s.replace(escaped_value, raw_value) return s @@ -151,10 +151,10 @@ def parse_function_call_and_arguments(function_name, function_call): >>> parse_function_call_and_arguments("strprintf", 'strprintf("%s (%d)", foo>foo<1,2>(1,2),err)'); ['strprintf(', '"%s (%d)",', ' foo>foo<1,2>(1,2),', 'err', ')'] """ - assert(type(function_name) is str and type(function_call) is str and function_name) + assert type(function_name) is str and type(function_call) is str and function_name remaining = normalize(escape(function_call)) expected_function_call = "{}(".format(function_name) - assert(remaining.startswith(expected_function_call)) + assert remaining.startswith(expected_function_call) parts = [expected_function_call] remaining = remaining[len(expected_function_call):] open_parentheses = 1 @@ -213,7 +213,7 @@ def parse_string_content(argument): >>> parse_string_content('1 2 3') '' """ - assert(type(argument) is str) + assert type(argument) is str string_content = "" in_string = False for char in normalize(escape(argument)): @@ -240,7 +240,7 @@ def count_format_specifiers(format_string): >>> count_format_specifiers("foo %d bar %i foo %% foo %*d foo") 4 """ - assert(type(format_string) is str) + assert type(format_string) is str format_string = format_string.replace('%%', 'X') n = 0 in_specifier = False @@ -262,27 +262,27 @@ def main(): parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string " "argument (e.g. 1 in the case of fprintf)", default=0) parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None) - parser.add_argument("file", type=argparse.FileType("r", encoding="utf-8"), nargs="*", help="C++ source code file (e.g. foo.cpp)") + parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)") args = parser.parse_args() - exit_code = 0 - for f in args.file: - for function_call_str in parse_function_calls(args.function_name, f.read()): - parts = parse_function_call_and_arguments(args.function_name, function_call_str) - relevant_function_call_str = unescape("".join(parts))[:512] - if (f.name, relevant_function_call_str) in FALSE_POSITIVES: - continue - if len(parts) < 3 + args.skip_arguments: - exit_code = 1 - print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) - continue - argument_count = len(parts) - 3 - args.skip_arguments - format_str = parse_string_content(parts[1 + args.skip_arguments]) - format_specifier_count = count_format_specifiers(format_str) - if format_specifier_count != argument_count: - exit_code = 1 - print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) - continue + for filename in args.file: + with open(filename, "r", encoding="utf-8") as f: + for function_call_str in parse_function_calls(args.function_name, f.read()): + parts = parse_function_call_and_arguments(args.function_name, function_call_str) + relevant_function_call_str = unescape("".join(parts))[:512] + if (f.name, relevant_function_call_str) in FALSE_POSITIVES: + continue + if len(parts) < 3 + args.skip_arguments: + exit_code = 1 + print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) + continue + argument_count = len(parts) - 3 - args.skip_arguments + format_str = parse_string_content(parts[1 + args.skip_arguments]) + format_specifier_count = count_format_specifiers(format_str) + if format_specifier_count != argument_count: + exit_code = 1 + print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) + continue sys.exit(exit_code) diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 1534d5ef68..2b6c78c2c8 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -8,6 +8,7 @@ KNOWN_VIOLATIONS=( "src/dbwrapper.cpp:.*vsnprintf" "src/httprpc.cpp.*trim" "src/init.cpp:.*atoi" + "src/init.cpp:.*fprintf" "src/qt/rpcconsole.cpp:.*atoi" "src/rest.cpp:.*strtol" "src/test/dbwrapper_tests.cpp:.*snprintf" @@ -18,7 +19,6 @@ KNOWN_VIOLATIONS=( "src/util/strencodings.cpp:.*strtoul" "src/util/strencodings.h:.*atoi" "src/util/system.cpp:.*atoi" - "src/util/system.cpp:.*fprintf" ) REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)" diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh index f318e19071..d5c1dee42d 100755 --- a/test/lint/lint-whitespace.sh +++ b/test/lint/lint-whitespace.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017 The Bitcoin Core developers +# Copyright (c) 2017-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -12,11 +12,11 @@ export LC_ALL=C while getopts "?" opt; do case $opt in ?) - echo "Usage: .lint-whitespace.sh [N]" - echo " TRAVIS_COMMIT_RANGE='<commit range>' .lint-whitespace.sh" - echo " .lint-whitespace.sh -?" + echo "Usage: $0 [N]" + echo " TRAVIS_COMMIT_RANGE='<commit range>' $0" + echo " $0 -?" echo "Checks unstaged changes, the previous N commits, or a commit range." - echo "TRAVIS_COMMIT_RANGE='47ba2c3...ee50c9e' .lint-whitespace.sh" + echo "TRAVIS_COMMIT_RANGE='47ba2c3...ee50c9e' $0" exit 0 ;; esac diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index f0107f1361..d55119b266 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -4,7 +4,6 @@ bool:wallet/wallet.cpp float-divide-by-zero:policy/fees.cpp float-divide-by-zero:validation.cpp float-divide-by-zero:wallet/wallet.cpp -nonnull-attribute:support/cleanse.cpp unsigned-integer-overflow:arith_uint256.h unsigned-integer-overflow:basic_string.h unsigned-integer-overflow:bench/bench.h |